You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by we...@apache.org on 2022/07/19 04:25:27 UTC

[dolphinscheduler] branch 3.0.0-prepare updated (ae4c3ab4e1 -> 6df8593add)

This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a change to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


    from ae4c3ab4e1 [Fix] Fix the JSONUtils tool class time zone problem #10282 (#10284)
     new 4ceb420873 Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe (#10479)
     new 81cadd15d2 Optimize MasterServer, add MasterRPCService (#10371)
     new 318a8e3ae0 [Feature][metrics] Add master, worker metrics (#10326)
     new 52815975bc Add some warning log in master (#10383)
     new 1def1729bc Add lombok
     new 9a4c7f876a Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe (#10479)
     new 9a59054655 Fix PeerTaskInstancePriorityQueue cannot contains method use taskInstanceId to check (#10491)
     new 90c87f0121 [Fix-10413] Fix Master startup failure the server still hang (#10500)
     new 9a28d32057 Remove the schedule thread in LowerWeightHostManager (#10310)
     new 3ab9ee13fc Optimize master log, use MDC to inject workflow instance id and task instance id in log (#10516)
     new fc1c1f6ad1 add CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST
     new c488a9f828 Fix compile error
     new db31deb54f [Bug] [Master] Worker failover will cause task cannot be failover (#10631)
     new 4b224ae2e5 Validate master/worker config (#10649)
     new 6c83967ebe [Improvement-10617] Add comment in slot check (#10618)
     new 3b923e5933 [Fix-10666] Workflow submit failed will still in memory and never retry (#10667)
     new 7500e99682 [Fix-10785] Fix state event handle error will not retry (#10786)
     new 04c47034d4 Add task prepare metrics
     new da03ef32e9 Fix compile error
     new 4fc9bce444 [Fix-10785] Fix state event handle error will not retry (#10786)
     new 71edaf41a2 [Fix-10842] Fix master/worker failover will cause status incorrect (#10839)
     new b259deb196 [Fix-10854] Fix database restart may lost task instance status (#10866)
     new 9ee20cffef [Fix-10827] Fix network error cause worker cannot send message to master (#10886)
     new 527ee472fb Catch exception when check state in StateWheelExecuteThread (#10908)
     new 812d7a8f26 Fix taskInstance's host is not worker nettyServer address (#10926)
     new 44ddb6908e Fix kill yarn job error when failover caused by doesn't set ProcessDefinition (#10948)
     new 17f0be5bfb Kill task when we do master failover to avoid task lost (#10997)
     new f3250bf5fa Fix worker cannot shutdown due to resource close failed or heart beat check failed (#10979)
     new 6df8593add Return status

The 29 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 docs/docs/en/guide/metrics/metrics.md              |  154 ++
 docs/img/metrics/metrics-datasource.png            |  Bin 0 -> 336581 bytes
 docs/img/metrics/metrics-master.png                |  Bin 0 -> 467778 bytes
 docs/img/metrics/metrics-worker.png                |  Bin 0 -> 403432 bytes
 .../apache/dolphinscheduler/alert/AlertServer.java |   23 +-
 .../src/main/resources/banner.txt                  |    2 +-
 .../api/service/impl/ExecutorServiceImpl.java      |   12 +-
 .../src/main/resources/application.yaml            |    3 +
 dolphinscheduler-api/src/main/resources/banner.txt |    2 +-
 .../apache/dolphinscheduler/common/Constants.java  |   52 +-
 .../dolphinscheduler/common/enums/StateEvent.java  |  123 -
 .../common/enums/StateEventType.java               |    2 +-
 .../enums/{Event.java => TaskEventType.java}       |    2 +-
 .../apache/dolphinscheduler/common/graph/DAG.java  |   11 +
 .../dolphinscheduler/common/model/TaskNode.java    |    2 +-
 .../common/storage/StorageOperate.java             |   32 +-
 .../thread/{Stopper.java => BaseDaemonThread.java} |   22 +-
 .../dolphinscheduler/common/thread/Stopper.java    |   34 +-
 .../common/thread/ThreadUtils.java                 |   26 +-
 .../dolphinscheduler/common/utils/FileUtils.java   |   29 +-
 .../dolphinscheduler/common/utils/LoggerUtils.java |   34 +-
 .../dolphinscheduler/common/utils/OSUtils.java     |   21 +-
 .../dao/entity/ProcessInstance.java                |   12 +-
 .../dao/utils/TaskInstanceUtils.java               |   79 +
 .../dao/utils/TaskInstanceUtilsTest.java           |   40 +-
 .../server/master/MasterServer.java                |  113 +-
 .../cache/ProcessInstanceExecCacheManager.java     |   10 +-
 .../impl/ProcessInstanceExecCacheManagerImpl.java  |   23 +-
 .../server/master/config/MasterConfig.java         |  249 +-
 .../master/consumer/TaskPriorityQueueConsumer.java |   85 +-
 .../master/dispatch/context/ExecutionContext.java  |   45 +-
 .../dispatch/executor/NettyExecutorManager.java    |    5 +-
 .../dispatch/host/LowerWeightHostManager.java      |  136 +-
 .../master/dispatch/host/assign/Selector.java      |    2 +-
 .../assign/Selector.java => event/StateEvent.java} |   35 +-
 .../StateEventHandleError.java}                    |   23 +-
 .../StateEventHandleException.java}                |   22 +-
 .../Selector.java => event/StateEventHandler.java} |   24 +-
 .../master/event/StateEventHandlerManager.java     |   37 +-
 .../server/master/event/TaskDelayEventHandler.java |  120 +
 .../master/event/TaskDispatchEventHandler.java     |   88 +
 .../TaskEventHandleError.java}                     |   21 +-
 .../TaskEventHandleException.java}                 |   21 +-
 .../Selector.java => event/TaskEventHandler.java}  |   22 +-
 .../event/TaskRejectByWorkerEventHandler.java      |   84 +
 .../master/event/TaskResultEventHandler.java       |  125 +
 .../master/event/TaskRetryStateEventHandler.java   |   47 +
 .../master/event/TaskRunningEventHandler.java      |  118 +
 .../server/master/event/TaskStateEventHandler.java |  115 +
 .../master/event/TaskTimeoutStateEventHandler.java |   64 +
 .../event/TaskWaitTaskGroupStateHandler.java       |   28 +-
 .../event/WorkflowBlockStateEventHandler.java      |   60 +
 .../server/master/event/WorkflowEvent.java         |   19 +-
 .../WorkflowEventHandleError.java}                 |   21 +-
 .../WorkflowEventHandleException.java}             |   21 +-
 .../WorkflowEventHandler.java}                     |   21 +-
 .../server/master/event/WorkflowEventQueue.java    |   48 +
 .../server/master/event/WorkflowEventType.java     |   11 +-
 .../master/event/WorkflowStartEventHandler.java    |   86 +
 .../master/event/WorkflowStateEventHandler.java    |   94 +
 .../event/WorkflowTimeoutStateEventHandler.java    |   39 +-
 .../MasterException.java}                          |   21 +-
 .../server/master/metrics/MasterServerMetrics.java |   53 +
 .../master/metrics/ProcessInstanceMetrics.java     |  126 +
 .../server/master/metrics/TaskMetrics.java         |  137 +
 .../master/processor/StateEventProcessor.java      |   18 +-
 .../master/processor/TaskEventProcessor.java       |   16 +-
 .../processor/TaskExecuteResponseProcessor.java    |   23 +-
 .../processor/TaskExecuteRunningProcessor.java     |    8 +-
 .../processor/TaskKillResponseProcessor.java       |    3 +-
 .../master/processor/TaskRecallProcessor.java      |    0
 .../processor/queue/StateEventResponseService.java |   44 +-
 .../server/master/processor/queue/TaskEvent.java   |  126 +-
 .../master/processor/queue/TaskEventService.java   |   30 +-
 .../processor/queue/TaskExecuteRunnable.java       |  104 +
 .../master/processor/queue/TaskExecuteThread.java  |  224 --
 .../processor/queue/TaskExecuteThreadPool.java     |   72 +-
 .../registry/MasterConnectionStateListener.java    |   62 +
 .../master/registry/MasterRegistryClient.java      |   96 +-
 .../registry/MasterRegistryDataListener.java       |    1 +
 .../server/master/registry/ServerNodeManager.java  |   22 +
 .../WorkerInfoChangeListener.java}                 |   20 +-
 .../MasterRPCServer.java}                          |  131 +-
 .../server/master/runner/EventExecuteService.java  |   28 +-
 .../master/runner/FailoverExecuteThread.java       |   24 +-
 .../master/runner/MasterSchedulerBootstrap.java    |  257 ++
 .../master/runner/MasterSchedulerService.java      |  253 --
 .../master/runner/StateWheelExecuteThread.java     |  313 ++-
 .../server/master/runner/WorkflowEventLooper.java  |  108 +
 ...uteThread.java => WorkflowExecuteRunnable.java} | 1065 ++++----
 .../master/runner/WorkflowExecuteThreadPool.java   |   68 +-
 .../WorkflowSubmitStatue.java}                     |   26 +-
 .../master/runner/task/BaseTaskProcessor.java      |   26 +-
 .../master/runner/task/CommonTaskProcessor.java    |   11 +-
 .../server/master/runner/task/TaskInstanceKey.java |   62 +-
 .../master/runner/task/TaskProcessorFactory.java   |   33 +-
 .../server/master/service/FailoverService.java     |  343 +--
 .../master/service/MasterFailoverService.java      |  281 +++
 .../master/service/WorkerFailoverService.java      |  267 ++
 .../src/main/resources/application.yaml            |   19 +-
 .../src/main/resources/banner.txt                  |    2 +-
 .../src/main/resources/logback-spring.xml          |    4 +-
 .../server/master/BlockingTaskTest.java            |    5 +-
 .../server/master/ConditionsTaskTest.java          |    3 +-
 .../server/master/DependentTaskTest.java           |    3 +-
 .../server/master/SubProcessTaskTest.java          |    3 +-
 .../server/master/SwitchTaskTest.java              |    3 +-
 .../ProcessInstanceExecCacheManagerImplTest.java   |   14 +-
 .../master/dispatch/ExecutionContextTestUtils.java |   19 +-
 .../master/dispatch/ExecutorDispatcherTest.java    |    6 +-
 .../executor/NettyExecutorManagerTest.java         |   29 +-
 .../master/processor/TaskAckProcessorTest.java     |   20 +-
 .../processor/queue/TaskResponseServiceTest.java   |   53 +-
 .../master/registry/MasterRegistryClientTest.java  |    7 -
 .../master/registry/ServerNodeManagerTest.java     |   60 -
 .../WorkflowExecuteRunnableTest.java}              |   34 +-
 .../runner/task/TaskProcessorFactoryTest.java      |    4 +-
 .../server/master/service/FailoverServiceTest.java |   45 +-
 .../dolphinscheduler/meter/MeterConfiguration.java |   12 +
 .../resources/grafana-demo/docker-compose.yaml     |    5 +-
 .../resources/grafana/DolphinSchedulerMaster.json  | 2607 ++++++++++++++++----
 .../resources/grafana/DolphinSchedulerWorker.json  | 1598 ++++++++++++
 .../remote/NettyRemotingClient.java                |    4 +-
 .../remote/command/BaseCommand.java                |   57 +
 .../dolphinscheduler/remote/command/Command.java   |    3 +
 .../remote/command/CommandHeader.java              |   47 +-
 .../remote/command/CommandType.java                |   26 +-
 .../remote/command/HostUpdateCommand.java          |    3 -
 .../remote/command/StateEventResponseCommand.java  |    2 +-
 ...equestCommand.java => TaskDispatchCommand.java} |   50 +-
 ...eAckCommand.java => TaskExecuteAckCommand.java} |   48 +-
 .../remote/command/TaskExecuteResponseCommand.java |  212 --
 ...kCommand.java => TaskExecuteResultCommand.java} |   97 +-
 ...mand.java => TaskExecuteRunningAckMessage.java} |    6 +-
 .../remote/command/TaskExecuteRunningCommand.java  |  101 +-
 .../remote/command/TaskKillRequestCommand.java     |   22 +-
 ...questCommand.java => TaskRejectAckCommand.java} |   42 +-
 .../remote/command/TaskRejectCommand.java          |    0
 .../remote/processor/NettyRemoteChannel.java       |    7 +-
 .../processor/StateEventCallbackService.java       |   31 +-
 .../remote/utils/ChannelUtils.java                 |   12 +-
 .../apache/dolphinscheduler/remote/utils/Host.java |    5 +-
 .../server/registry/HeartBeatTask.java             |   21 +-
 .../server/utils/ProcessUtils.java                 |    7 +-
 .../service/alert/AlertClientService.java          |   21 +-
 .../service/bean/SpringApplicationContext.java     |    3 +-
 .../service/process/ProcessService.java            |   13 +-
 .../service/process/ProcessServiceImpl.java        |   84 +-
 .../queue/PeerTaskInstancePriorityQueue.java       |   56 +-
 .../service/queue/TaskPriority.java                |   41 +-
 .../service/queue/TaskPriorityQueue.java           |    2 +-
 .../service/queue/TaskPriorityQueueImpl.java       |   12 +-
 .../service/registry/RegistryClient.java           |    3 +-
 .../service/task/TaskPluginManager.java            |    5 +-
 .../service/process/ProcessServiceTest.java        |   61 +-
 .../queue/PeerTaskInstancePriorityQueueTest.java   |   15 +-
 .../apache/dolphinscheduler/StandaloneServer.java  |   32 +-
 .../src/main/resources/application.yaml            |   26 +-
 .../src/main/resources/banner.txt                  |    2 +-
 .../src/main/resources/logback-spring.xml          |    4 +-
 .../plugin/task/api/ProcessUtils.java              |   15 +-
 .../plugin/task/api/TaskExecutionContext.java      |  383 +--
 .../plugin/task/api/TaskPluginException.java       |   13 +-
 .../plugin/task/api/enums/ExecutionStatus.java     |   23 +
 .../plugin/task/shell/ShellTask.java               |    7 +-
 .../server/worker/WorkerServer.java                |  146 +-
 .../server/worker/cache/ResponseCache.java         |   98 -
 .../server/worker/config/WorkerConfig.java         |  154 +-
 .../server/worker/message/MessageRetryRunner.java  |  139 ++
 .../server/worker/message/MessageSender.java       |   43 +
 .../message/TaskExecuteResultMessageSender.java    |   70 +
 .../message/TaskExecuteRunningMessageSender.java   |   67 +
 .../worker/message/TaskRejectMessageSender.java    |   59 +
 .../server/worker/metrics/TaskMetrics.java         |   58 +
 .../server/worker/metrics/WorkerServerMetrics.java |   55 +
 .../worker/processor/HostUpdateProcessor.java      |   13 +-
 .../worker/processor/TaskCallbackService.java      |  260 --
 .../worker/processor/TaskDispatchProcessor.java    |  212 ++
 .../worker/processor/TaskExecuteProcessor.java     |  186 --
 .../processor/TaskExecuteResponseAckProcessor.java |   65 -
 .../processor/TaskExecuteResultAckProcessor.java   |   83 +
 .../processor/TaskExecuteRunningAckProcessor.java  |   28 +-
 .../server/worker/processor/TaskKillProcessor.java |   49 +-
 .../worker/processor/TaskRejectAckProcessor.java   |    0
 .../worker/registry/WorkerRegistryClient.java      |   38 +-
 .../server/worker/rpc/WorkerMessageSender.java     |   92 +
 .../server/worker/rpc/WorkerRpcClient.java         |   70 +
 .../server/worker/rpc/WorkerRpcServer.java         |   97 +
 .../worker/runner/RetryReportTaskStatusThread.java |   90 -
 .../server/worker/runner/TaskExecuteThread.java    |  126 +-
 .../server/worker/runner/WorkerExecService.java    |    9 +-
 .../server/worker/runner/WorkerManagerThread.java  |   39 +-
 .../src/main/resources/application.yaml            |    9 +-
 .../src/main/resources/banner.txt                  |    2 +-
 .../src/main/resources/logback-spring.xml          |    4 +-
 ...sorTest.java => TaskDispatchProcessorTest.java} |  100 +-
 .../worker/registry/WorkerRegistryClientTest.java  |   13 +-
 .../worker/runner/TaskExecuteThreadTest.java       |   35 +-
 pom.xml                                            |    7 +
 199 files changed, 10414 insertions(+), 5582 deletions(-)
 create mode 100644 docs/docs/en/guide/metrics/metrics.md
 create mode 100644 docs/img/metrics/metrics-datasource.png
 create mode 100644 docs/img/metrics/metrics-master.png
 create mode 100644 docs/img/metrics/metrics-worker.png
 delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
 copy dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/{Event.java => TaskEventType.java} (97%)
 copy dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/{Stopper.java => BaseDaemonThread.java} (65%)
 create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java
 rename dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java => dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java (50%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/StateEvent.java} (56%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/StateEventHandleError.java} (63%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/StateEventHandleException.java} (65%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/StateEventHandler.java} (51%)
 copy dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java => dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java (52%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDispatchEventHandler.java
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/TaskEventHandleError.java} (72%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/TaskEventHandleException.java} (71%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/TaskEventHandler.java} (58%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java
 copy dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java => dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java (55%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java
 copy dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java => dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEvent.java (75%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/WorkflowEventHandleError.java} (71%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/WorkflowEventHandleException.java} (71%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => event/WorkflowEventHandler.java} (60%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventQueue.java
 copy dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java => dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventType.java (87%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java
 copy dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java => dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java (50%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => exception/MasterException.java} (73%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java
 copy dolphinscheduler-python/pydolphinscheduler/docs/source/_static/.gitkeep => dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java (100%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
 delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThread.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => registry/WorkerInfoChangeListener.java} (58%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{MasterServer.java => rpc/MasterRPCServer.java} (50%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
 delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java
 rename dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/{WorkflowExecuteThread.java => WorkflowExecuteRunnable.java} (66%)
 copy dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/{dispatch/host/assign/Selector.java => runner/WorkflowSubmitStatue.java} (72%)
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
 create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java
 delete mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java
 rename dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/{WorkflowExecuteThreadTest.java => runner/WorkflowExecuteRunnableTest.java} (90%)
 create mode 100644 dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerWorker.json
 create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseCommand.java
 rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/{TaskExecuteRequestCommand.java => TaskDispatchCommand.java} (57%)
 copy dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/{TaskExecuteResponseAckCommand.java => TaskExecuteAckCommand.java} (62%)
 delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java
 rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/{TaskExecuteResponseAckCommand.java => TaskExecuteResultCommand.java} (51%)
 rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/{TaskExecuteRunningAckCommand.java => TaskExecuteRunningAckMessage.java} (92%)
 copy dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/{TaskKillRequestCommand.java => TaskRejectAckCommand.java} (62%)
 copy dolphinscheduler-python/pydolphinscheduler/docs/source/_static/.gitkeep => dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRejectCommand.java (100%)
 rename dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java => dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginException.java (80%)
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageSender.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteRunningMessageSender.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java
 delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
 delete mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java
 copy dolphinscheduler-python/pydolphinscheduler/docs/source/_static/.gitkeep => dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java (100%)
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java
 create mode 100644 dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java
 rename dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/{TaskExecuteProcessorTest.java => TaskDispatchProcessorTest.java} (58%)


[dolphinscheduler] 24/29: Catch exception when check state in StateWheelExecuteThread (#10908)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 527ee472fbc22cee6cd564f2cb6e3ee008939d04
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Wed Jul 13 10:51:20 2022 +0800

    Catch exception when check state in StateWheelExecuteThread (#10908)
    
    * Catch exception when check state
    
    (cherry picked from commit 2a67866718a74a6abc30fb615c0e16978511e3eb)
---
 .../master/runner/StateWheelExecuteThread.java     | 68 ++++++++++++++--------
 .../master/runner/WorkflowExecuteRunnable.java     |  5 +-
 .../src/main/resources/application.yaml            |  2 +-
 .../service/process/ProcessServiceImpl.java        |  2 +-
 .../service/process/ProcessServiceTest.java        |  7 ++-
 .../src/main/resources/application.yaml            |  2 +-
 6 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 0d1b3a423a..c053cb238b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -134,23 +134,34 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
             return;
         }
         for (Integer processInstanceId : processInstanceTimeoutCheckList) {
-            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
-            if (workflowExecuteThread == null) {
-                logger.warn("Check workflow timeout failed, can not find workflowExecuteThread from cache manager, will remove this workflowInstance from check list");
-                processInstanceTimeoutCheckList.remove(processInstanceId);
-                continue;
-            }
-            ProcessInstance processInstance = workflowExecuteThread.getProcessInstance();
-            if (processInstance == null) {
-                logger.warn("Check workflow timeout failed, the workflowInstance is null");
-                continue;
-            }
-            long timeRemain = DateUtils.getRemainTime(processInstance.getStartTime(), (long) processInstance.getTimeout() * Constants.SEC_2_MINUTES_TIME_UNIT);
-            if (timeRemain < 0) {
-                logger.info("Workflow instance timeout, adding timeout event");
-                addProcessTimeoutEvent(processInstance);
-                processInstanceTimeoutCheckList.remove(processInstance.getId());
-                logger.info("Workflow instance timeout, added timeout event");
+            try {
+                LoggerUtils.setWorkflowInstanceIdMDC(processInstanceId);
+                WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(
+                    processInstanceId);
+                if (workflowExecuteThread == null) {
+                    logger.warn(
+                        "Check workflow timeout failed, can not find workflowExecuteThread from cache manager, will remove this workflowInstance from check list");
+                    processInstanceTimeoutCheckList.remove(processInstanceId);
+                    continue;
+                }
+                ProcessInstance processInstance = workflowExecuteThread.getProcessInstance();
+                if (processInstance == null) {
+                    logger.warn("Check workflow timeout failed, the workflowInstance is null");
+                    continue;
+                }
+                long timeRemain = DateUtils.getRemainTime(processInstance.getStartTime(),
+                                                          (long) processInstance.getTimeout()
+                                                              * Constants.SEC_2_MINUTES_TIME_UNIT);
+                if (timeRemain < 0) {
+                    logger.info("Workflow instance timeout, adding timeout event");
+                    addProcessTimeoutEvent(processInstance);
+                    processInstanceTimeoutCheckList.remove(processInstance.getId());
+                    logger.info("Workflow instance timeout, added timeout event");
+                }
+            } catch (Exception ex) {
+                logger.error("Check workflow instance timeout error");
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
             }
         }
     }
@@ -243,20 +254,26 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
                 }
                 Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
                 if (!taskInstanceOptional.isPresent()) {
-                    logger.warn("Check task instance timeout failed, can not get taskInstance from workflowExecuteThread, taskCode: {}"
-                        + "will remove this check task", taskCode);
+                    logger.warn(
+                        "Check task instance timeout failed, can not get taskInstance from workflowExecuteThread, taskCode: {}"
+                            + "will remove this check task",
+                        taskCode);
                     taskInstanceTimeoutCheckList.remove(taskInstanceKey);
                     continue;
                 }
                 TaskInstance taskInstance = taskInstanceOptional.get();
                 if (TimeoutFlag.OPEN == taskInstance.getTaskDefine().getTimeoutFlag()) {
-                    long timeRemain = DateUtils.getRemainTime(taskInstance.getStartTime(), (long) taskInstance.getTaskDefine().getTimeout() * Constants.SEC_2_MINUTES_TIME_UNIT);
+                    long timeRemain = DateUtils.getRemainTime(taskInstance.getStartTime(),
+                                                              (long) taskInstance.getTaskDefine().getTimeout()
+                                                                  * Constants.SEC_2_MINUTES_TIME_UNIT);
                     if (timeRemain < 0) {
                         logger.info("Task instance is timeout, adding task timeout event and remove the check");
                         addTaskTimeoutEvent(taskInstance);
                         taskInstanceTimeoutCheckList.remove(taskInstanceKey);
                     }
                 }
+            } catch (Exception ex) {
+                logger.error("Check task timeout error, taskInstanceKey: {}", taskInstanceKey, ex);
             } finally {
                 LoggerUtils.removeWorkflowInstanceIdMDC();
             }
@@ -277,8 +294,9 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
                 WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
 
                 if (workflowExecuteThread == null) {
-                    logger.warn("Task instance retry check failed, can not find workflowExecuteThread from cache manager, "
-                        + "will remove this check task");
+                    logger.warn(
+                        "Task instance retry check failed, can not find workflowExecuteThread from cache manager, "
+                            + "will remove this check task");
                     taskInstanceRetryCheckList.remove(taskInstanceKey);
                     continue;
                 }
@@ -308,13 +326,15 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
                     // reset taskInstance endTime and state
                     // todo relative funtion: TaskInstance.retryTaskIntervalOverTime, WorkflowExecuteThread.cloneRetryTaskInstance
                     logger.info("[TaskInstance-{}]The task instance can retry, will retry this task instance",
-                        taskInstance.getId());
+                                taskInstance.getId());
                     taskInstance.setEndTime(null);
                     taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
 
                     addTaskRetryEvent(taskInstance);
                     taskInstanceRetryCheckList.remove(taskInstanceKey);
                 }
+            } catch (Exception ex) {
+                logger.error("Check task retry error, taskInstanceKey: {}", taskInstanceKey, ex);
             } finally {
                 LoggerUtils.removeWorkflowInstanceIdMDC();
             }
@@ -349,6 +369,8 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
                     continue;
                 }
                 addTaskStateChangeEvent(taskInstance);
+            } catch (Exception ex) {
+                logger.error("Task state check error, taskInstanceKey: {}", taskInstanceKey, ex);
             } finally {
                 LoggerUtils.removeWorkflowInstanceIdMDC();
             }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index a433981aca..0e94b0318d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -552,8 +552,9 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     }
 
     public Optional<TaskInstance> getActiveTaskInstanceByTaskCode(long taskCode) {
-        if (activeTaskProcessorMaps.containsKey(taskCode)) {
-            return Optional.ofNullable(activeTaskProcessorMaps.get(taskCode).taskInstance());
+        Integer taskInstanceId = validTaskMap.get(taskCode);
+        if (taskInstanceId != null) {
+            return Optional.ofNullable(taskInstanceMap.get(taskInstanceId));
         }
         return Optional.empty();
     }
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index a277272026..ff29b19389 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -102,7 +102,7 @@ master:
   task-commit-retry-times: 5
   # master commit task interval
   task-commit-interval: 1s
-  state-wheel-interval: 5
+  state-wheel-interval: 5s
   # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
   max-cpu-load-avg: -1
   # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index b12cd77de2..9cce9152f2 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -909,7 +909,7 @@ public class ProcessServiceImpl implements ProcessService {
                                                        command.getProcessDefinitionVersion());
         if (processDefinition == null) {
             logger.error("cannot find the work process define! define code : {}", command.getProcessDefinitionCode());
-            return null;
+            throw new IllegalArgumentException("Cannot find the process definition for this workflowInstance");
         }
         Map<String, String> cmdParam = JSONUtils.toMap(command.getCommandParam());
         int processInstanceId = command.getProcessInstanceId();
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
index a98138dd03..82dc3f4673 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
@@ -292,7 +292,12 @@ public class ProcessServiceTest {
                                     + "\":\"111\",\""
                                     + CMD_PARAM_SUB_PROCESS_DEFINE_CODE
                                     + "\":\"222\"}");
-        Assert.assertNull(processService.handleCommand(host, command));
+        try {
+            Assert.assertNull(processService.handleCommand(host, command));
+        } catch (IllegalArgumentException illegalArgumentException) {
+            // assert throw illegalArgumentException here since the definition is null
+            Assert.assertTrue(true);
+        }
 
         int definitionVersion = 1;
         long definitionCode = 123;
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index cf5ce5ea88..9a940b6c0e 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -120,7 +120,7 @@ master:
   task-commit-retry-times: 5
   # master commit task interval
   task-commit-interval: 1s
-  state-wheel-interval: 5
+  state-wheel-interval: 5s
   # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
   max-cpu-load-avg: -1
   # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G


[dolphinscheduler] 21/29: [Fix-10842] Fix master/worker failover will cause status incorrect (#10839)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 71edaf41a234f5c73e0c6266b6b8b927b146cd89
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Sat Jul 9 11:54:59 2022 +0800

    [Fix-10842] Fix master/worker failover will cause status incorrect (#10839)
    
    * Fix master failover will not update task instance status
    * Add some failover log
    * Fix worker failover will rerun task more than once
    * Fix workflowInstance failover may rerun already success taskInstance
    
    (cherry picked from commit 3f69ec8f28c7206c153f5de8c7c0d1fa33d311f9)
---
 .../apache/dolphinscheduler/common/graph/DAG.java  |  11 +
 .../master/consumer/TaskPriorityQueueConsumer.java |  32 +-
 .../master/dispatch/context/ExecutionContext.java  |  45 +--
 .../dispatch/executor/NettyExecutorManager.java    |   3 +
 .../processor/queue/TaskExecuteRunnable.java       | 124 +++----
 .../registry/MasterRegistryDataListener.java       |   1 +
 .../master/runner/FailoverExecuteThread.java       |   5 +-
 .../master/runner/StateWheelExecuteThread.java     |  12 +-
 .../master/runner/WorkflowExecuteRunnable.java     | 115 ++++---
 .../master/runner/task/CommonTaskProcessor.java    |   5 +-
 .../server/master/service/FailoverService.java     | 371 +--------------------
 .../master/service/MasterFailoverService.java      | 253 ++++++++++++++
 .../master/service/WorkerFailoverService.java      | 266 +++++++++++++++
 .../master/dispatch/ExecutionContextTestUtils.java |   2 +-
 .../executor/NettyExecutorManagerTest.java         |   4 +-
 .../server/master/service/FailoverServiceTest.java |  22 +-
 .../service/process/ProcessService.java            |   1 +
 .../service/process/ProcessServiceImpl.java        |  38 ++-
 .../service/queue/TaskPriority.java                |  41 ++-
 19 files changed, 823 insertions(+), 528 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java
index e57b3dd93e..afa780163a 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java
@@ -498,5 +498,16 @@ public class DAG<Node, NodeInfo, EdgeInfo> {
         return new AbstractMap.SimpleEntry<>(notZeroIndegreeNodeMap.size() == 0, topoResultList);
     }
 
+    @Override
+    public String toString() {
+        return "DAG{"
+            + "nodesMap="
+            + nodesMap
+            + ", edgesMap="
+            + edgesMap
+            + ", reverseEdgesMap="
+            + reverseEdgesMap
+            + '}';
+    }
 }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index f04ae15dd8..82198e942a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -34,11 +34,11 @@ import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteExce
 import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.service.exceptions.TaskPriorityQueueException;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.queue.TaskPriority;
 import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
-import org.apache.dolphinscheduler.spi.utils.JSONUtils;
 
 import org.apache.commons.collections.CollectionUtils;
 
@@ -46,6 +46,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -187,8 +188,24 @@ public class TaskPriorityQueueConsumer extends BaseDaemonThread {
         TaskMetrics.incTaskDispatch();
         boolean result = false;
         try {
+            WorkflowExecuteRunnable workflowExecuteRunnable =
+                processInstanceExecCacheManager.getByProcessInstanceId(taskPriority.getProcessInstanceId());
+            if (workflowExecuteRunnable == null) {
+                logger.error("Cannot find the related processInstance of the task, taskPriority: {}", taskPriority);
+                return true;
+            }
+            Optional<TaskInstance> taskInstanceOptional =
+                workflowExecuteRunnable.getTaskInstance(taskPriority.getTaskId());
+            if (!taskInstanceOptional.isPresent()) {
+                logger.error("Cannot find the task instance from related processInstance, taskPriority: {}",
+                    taskPriority);
+                // we return true, so that we will drop this task.
+                return true;
+            }
+            TaskInstance taskInstance = taskInstanceOptional.get();
             TaskExecutionContext context = taskPriority.getTaskExecutionContext();
-            ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, context.getWorkerGroup());
+            ExecutionContext executionContext =
+                new ExecutionContext(toCommand(context), ExecutorType.WORKER, context.getWorkerGroup(), taskInstance);
 
             if (isTaskNeedToCheck(taskPriority)) {
                 if (taskInstanceIsFinalState(taskPriority.getTaskId())) {
@@ -196,16 +213,21 @@ public class TaskPriorityQueueConsumer extends BaseDaemonThread {
                     return true;
                 }
             }
+
             result = dispatcher.dispatch(executionContext);
 
             if (result) {
-                logger.info("Master success dispatch task to worker, taskInstanceId: {}", taskPriority.getTaskId());
+                logger.info("Master success dispatch task to worker, taskInstanceId: {}, worker: {}",
+                    taskPriority.getTaskId(),
+                    executionContext.getHost());
                 addDispatchEvent(context, executionContext);
             } else {
-                logger.info("Master failed to dispatch task to worker, taskInstanceId: {}", taskPriority.getTaskId());
+                logger.info("Master failed to dispatch task to worker, taskInstanceId: {}, worker: {}",
+                    taskPriority.getTaskId(),
+                    executionContext.getHost());
             }
         } catch (RuntimeException | ExecuteException e) {
-            logger.error("Master dispatch task to worker error: ", e);
+            logger.error("Master dispatch task to worker error, taskPriority: {}", taskPriority, e);
         }
         return result;
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
index b3fba87870..880640e91d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/context/ExecutionContext.java
@@ -19,13 +19,17 @@ package org.apache.dolphinscheduler.server.master.dispatch.context;
 
 import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
 
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
 
+import lombok.Data;
+
 /**
  *  execution context
  */
+@Data
 public class ExecutionContext {
 
     /**
@@ -34,51 +38,30 @@ public class ExecutionContext {
     private Host host;
 
     /**
-     *  command
+     * command
      */
     private final Command command;
 
+    private final TaskInstance taskInstance;
+
     /**
-     *  executor type : worker or client
+     * executor type : worker or client
      */
     private final ExecutorType executorType;
 
     /**
-     *  worker group
+     * worker group
      */
-    private String workerGroup;
+    private final String workerGroup;
 
-    public ExecutionContext(Command command, ExecutorType executorType) {
-        this(command, executorType, DEFAULT_WORKER_GROUP);
+    public ExecutionContext(Command command, ExecutorType executorType, TaskInstance taskInstance) {
+        this(command, executorType, DEFAULT_WORKER_GROUP, taskInstance);
     }
 
-    public ExecutionContext(Command command, ExecutorType executorType, String workerGroup) {
+    public ExecutionContext(Command command, ExecutorType executorType, String workerGroup, TaskInstance taskInstance) {
         this.command = command;
         this.executorType = executorType;
         this.workerGroup = workerGroup;
-    }
-
-    public Command getCommand() {
-        return command;
-    }
-
-    public ExecutorType getExecutorType() {
-        return executorType;
-    }
-
-    public void setWorkerGroup(String workerGroup) {
-        this.workerGroup = workerGroup;
-    }
-
-    public String getWorkerGroup() {
-        return this.workerGroup;
-    }
-
-    public Host getHost() {
-        return host;
-    }
-
-    public void setHost(Host host) {
-        this.host = host;
+        this.taskInstance = taskInstance;
     }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
index 0ba24e287d..0cc2d4b8f7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
@@ -123,6 +123,9 @@ public class NettyExecutorManager extends AbstractExecutorManager<Boolean> {
                 doExecute(host, command);
                 success = true;
                 context.setHost(host);
+                // We set the host to taskInstance to avoid when the worker down, this taskInstance may not be failovered, due to the taskInstance's host
+                // is not belongs to the down worker ISSUE-10842.
+                context.getTaskInstance().setHost(host.getAddress());
             } catch (ExecuteException ex) {
                 logger.error(String.format("execute command : %s error", command), ex);
                 try {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
index f488fc3de1..d57adcbbbc 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
@@ -18,7 +18,6 @@
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
 import org.apache.dolphinscheduler.common.enums.Event;
-import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
 import org.apache.dolphinscheduler.server.utils.DataQualityResultOperator;
@@ -75,7 +75,7 @@ public class TaskExecuteRunnable implements Runnable {
                 LoggerUtils.setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), event.getTaskInstanceId());
                 persist(event);
             } catch (Exception e) {
-                logger.error("persist error, event:{}, error: {}", event, e);
+                logger.error("persist task event error, event:{}", event, e);
             } finally {
                 this.events.remove(event);
                 LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
@@ -113,27 +113,29 @@ public class TaskExecuteRunnable implements Runnable {
      *
      * @param taskEvent taskEvent
      */
-    private void persist(TaskEvent taskEvent) {
+    private void persist(TaskEvent taskEvent) throws Exception {
         Event event = taskEvent.getEvent();
         int taskInstanceId = taskEvent.getTaskInstanceId();
         int processInstanceId = taskEvent.getProcessInstanceId();
 
         Optional<TaskInstance> taskInstance;
-        WorkflowExecuteRunnable workflowExecuteRunnable = this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        WorkflowExecuteRunnable workflowExecuteRunnable =
+            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
         if (workflowExecuteRunnable != null && workflowExecuteRunnable.checkTaskInstanceById(taskInstanceId)) {
             taskInstance = workflowExecuteRunnable.getTaskInstance(taskInstanceId);
         } else {
             taskInstance = Optional.ofNullable(processService.findTaskInstanceById(taskInstanceId));
         }
 
+        boolean needToSendEvent = true;
         switch (event) {
             case DISPATCH:
-                handleDispatchEvent(taskEvent, taskInstance);
+                needToSendEvent = handleDispatchEvent(taskEvent, taskInstance);
                 // dispatch event do not need to submit state event
-                return;
+                break;
             case DELAY:
             case RUNNING:
-                handleRunningEvent(taskEvent, taskInstance);
+                needToSendEvent = handleRunningEvent(taskEvent, taskInstance);
                 break;
             case RESULT:
                 handleResultEvent(taskEvent, taskInstance);
@@ -141,6 +143,10 @@ public class TaskExecuteRunnable implements Runnable {
             default:
                 throw new IllegalArgumentException("invalid event type : " + event);
         }
+        if (!needToSendEvent) {
+            logger.info("Handle task event: {} success, there is no need to send a StateEvent", taskEvent);
+            return;
+        }
 
         StateEvent stateEvent = new StateEvent();
         stateEvent.setProcessInstanceId(taskEvent.getProcessInstanceId());
@@ -153,81 +159,83 @@ public class TaskExecuteRunnable implements Runnable {
     /**
      * handle dispatch event
      */
-    private void handleDispatchEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
+    private boolean handleDispatchEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
         if (!taskInstanceOptional.isPresent()) {
             logger.error("taskInstance is null");
-            return;
+            return false;
         }
         TaskInstance taskInstance = taskInstanceOptional.get();
         if (taskInstance.getState() != ExecutionStatus.SUBMITTED_SUCCESS) {
-            return;
+            return false;
         }
         taskInstance.setState(ExecutionStatus.DISPATCH);
         taskInstance.setHost(taskEvent.getWorkerAddress());
         processService.saveTaskInstance(taskInstance);
+        return true;
     }
 
     /**
      * handle running event
      */
-    private void handleRunningEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
+    private boolean handleRunningEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
         Channel channel = taskEvent.getChannel();
-        try {
-            if (taskInstanceOptional.isPresent()) {
-                TaskInstance taskInstance = taskInstanceOptional.get();
-                if (taskInstance.getState().typeIsFinished()) {
-                    logger.warn("task is finish, running event is meaningless, taskInstanceId:{}, state:{}", taskInstance.getId(), taskInstance.getState());
-                } else {
-                    taskInstance.setState(taskEvent.getState());
-                    taskInstance.setStartTime(taskEvent.getStartTime());
-                    taskInstance.setHost(taskEvent.getWorkerAddress());
-                    taskInstance.setLogPath(taskEvent.getLogPath());
-                    taskInstance.setExecutePath(taskEvent.getExecutePath());
-                    taskInstance.setPid(taskEvent.getProcessId());
-                    taskInstance.setAppLink(taskEvent.getAppIds());
-                    processService.saveTaskInstance(taskInstance);
-                }
+        if (taskInstanceOptional.isPresent()) {
+            TaskInstance taskInstance = taskInstanceOptional.get();
+            if (taskInstance.getState().typeIsFinished()) {
+                logger.warn("task is finish, running event is meaningless, taskInstanceId:{}, state:{}",
+                    taskInstance.getId(),
+                    taskInstance.getState());
+                return false;
+            } else {
+                taskInstance.setState(taskEvent.getState());
+                taskInstance.setStartTime(taskEvent.getStartTime());
+                taskInstance.setHost(taskEvent.getWorkerAddress());
+                taskInstance.setLogPath(taskEvent.getLogPath());
+                taskInstance.setExecutePath(taskEvent.getExecutePath());
+                taskInstance.setPid(taskEvent.getProcessId());
+                taskInstance.setAppLink(taskEvent.getAppIds());
+                processService.saveTaskInstance(taskInstance);
             }
-            // if taskInstance is null (maybe deleted) or finish. retry will be meaningless . so ack success
-            // send ack to worker
-            TaskExecuteRunningAckCommand taskExecuteRunningAckCommand = new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-            channel.writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
-        } catch (Exception e) {
-            logger.error("worker ack master error", e);
-            TaskExecuteRunningAckCommand taskExecuteRunningAckCommand = new TaskExecuteRunningAckCommand(ExecutionStatus.FAILURE.getCode(), -1);
-            channel.writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
         }
+        // if taskInstance is null (maybe deleted) or finish. retry will be meaningless . so ack success
+        // send ack to worker
+        TaskExecuteRunningAckCommand taskExecuteRunningAckCommand =
+            new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        channel.writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
+        return true;
     }
 
     /**
      * handle result event
      */
-    private void handleResultEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
+    private boolean handleResultEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
         Channel channel = taskEvent.getChannel();
-        try {
-            if (taskInstanceOptional.isPresent()) {
-                TaskInstance taskInstance = taskInstanceOptional.get();
-                dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance);
-
-                taskInstance.setStartTime(taskEvent.getStartTime());
-                taskInstance.setHost(taskEvent.getWorkerAddress());
-                taskInstance.setLogPath(taskEvent.getLogPath());
-                taskInstance.setExecutePath(taskEvent.getExecutePath());
-                taskInstance.setPid(taskEvent.getProcessId());
-                taskInstance.setAppLink(taskEvent.getAppIds());
-                taskInstance.setState(taskEvent.getState());
-                taskInstance.setEndTime(taskEvent.getEndTime());
-                taskInstance.setVarPool(taskEvent.getVarPool());
-                processService.changeOutParam(taskInstance);
-                processService.saveTaskInstance(taskInstance);
+        if (taskInstanceOptional.isPresent()) {
+            TaskInstance taskInstance = taskInstanceOptional.get();
+            if (taskInstance.getState().typeIsFinished()) {
+                logger.warn("The current taskInstance has already been finished, taskEvent: {}", taskEvent);
+                return false;
             }
-            // if taskInstance is null (maybe deleted) . retry will be meaningless . so response success
-            TaskExecuteResponseAckCommand taskExecuteResponseAckCommand = new TaskExecuteResponseAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-            channel.writeAndFlush(taskExecuteResponseAckCommand.convert2Command());
-        } catch (Exception e) {
-            logger.error("worker response master error", e);
-            TaskExecuteResponseAckCommand taskExecuteResponseAckCommand = new TaskExecuteResponseAckCommand(ExecutionStatus.FAILURE.getCode(), -1);
-            channel.writeAndFlush(taskExecuteResponseAckCommand.convert2Command());
+
+            dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance);
+
+            taskInstance.setStartTime(taskEvent.getStartTime());
+            taskInstance.setHost(taskEvent.getWorkerAddress());
+            taskInstance.setLogPath(taskEvent.getLogPath());
+            taskInstance.setExecutePath(taskEvent.getExecutePath());
+            taskInstance.setPid(taskEvent.getProcessId());
+            taskInstance.setAppLink(taskEvent.getAppIds());
+            taskInstance.setState(taskEvent.getState());
+            taskInstance.setEndTime(taskEvent.getEndTime());
+            taskInstance.setVarPool(taskEvent.getVarPool());
+            processService.changeOutParam(taskInstance);
+            processService.saveTaskInstance(taskInstance);
         }
+        // if taskInstance is null (maybe deleted) . retry will be meaningless . so response success
+        TaskExecuteResponseAckCommand taskExecuteResponseAckCommand =
+            new TaskExecuteResponseAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        channel.writeAndFlush(taskExecuteResponseAckCommand.convert2Command());
+        return true;
     }
+
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
index 30cea36db7..1eafb4cb54 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
@@ -64,6 +64,7 @@ public class MasterRegistryDataListener implements SubscribeListener {
                 break;
             case REMOVE:
                 masterRegistryClient.removeMasterNodePath(path, NodeType.MASTER, true);
+
                 break;
             default:
                 break;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
index 63f4215f27..16656c8760 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
@@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.service.FailoverService;
+import org.apache.dolphinscheduler.server.master.service.MasterFailoverService;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -41,7 +42,7 @@ public class FailoverExecuteThread extends BaseDaemonThread {
      * failover service
      */
     @Autowired
-    private FailoverService failoverService;
+    private MasterFailoverService masterFailoverService;
 
     protected FailoverExecuteThread() {
         super("FailoverExecuteThread");
@@ -63,7 +64,7 @@ public class FailoverExecuteThread extends BaseDaemonThread {
             try {
                 // todo: DO we need to schedule a task to do this kind of check
                 // This kind of check may only need to be executed when a master server start
-                failoverService.checkMasterFailover();
+                masterFailoverService.checkMasterFailover();
             } catch (Exception e) {
                 logger.error("Master failover thread execute error", e);
             } finally {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index c70fb1f1d4..843c1b246c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -18,7 +18,6 @@
 package org.apache.dolphinscheduler.server.master.runner;
 
 import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
@@ -31,6 +30,7 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
 
 import java.util.Optional;
@@ -296,15 +296,21 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
                 }
 
                 if (!taskInstanceOptional.isPresent()) {
-                    logger.warn("Task instance retry check failed, can not find taskInstance from workflowExecuteThread, will remove this check");
+                    logger.warn(
+                        "Task instance retry check failed, can not find taskInstance from workflowExecuteThread, will remove this check");
                     taskInstanceRetryCheckList.remove(taskInstanceKey);
                     continue;
                 }
 
                 TaskInstance taskInstance = taskInstanceOptional.get();
-                if (taskInstance.retryTaskIntervalOverTime()) {
+                // We check the status to avoid when we do worker failover we submit a failover task, this task may be resubmit by this
+                // thread
+                if (taskInstance.getState() != ExecutionStatus.NEED_FAULT_TOLERANCE
+                    && taskInstance.retryTaskIntervalOverTime()) {
                     // reset taskInstance endTime and state
                     // todo relative funtion: TaskInstance.retryTaskIntervalOverTime, WorkflowExecuteThread.cloneRetryTaskInstance
+                    logger.info("[TaskInstance-{}]The task instance can retry, will retry this task instance",
+                        taskInstance.getId());
                     taskInstance.setEndTime(null);
                     taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 16604662ac..2763d99932 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -107,6 +107,7 @@ import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanUtils;
 
 import com.google.common.collect.Lists;
 
@@ -140,7 +141,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     /**
      * process instance
      */
-    private ProcessInstance processInstance;
+    private final ProcessInstance processInstance;
 
     /**
      * process definition
@@ -289,6 +290,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
                 StateEventHandler stateEventHandler =
                     StateEventHandlerManager.getStateEventHandler(stateEvent.getType())
                         .orElseThrow(() -> new StateEventHandleError("Cannot find handler for the given state event"));
+                logger.info("Begin to handle state event, {}", stateEvent);
                 if (stateEventHandler.handleStateEvent(this, stateEvent)) {
                     this.stateEvents.remove(stateEvent);
                 }
@@ -482,9 +484,12 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      */
     public void refreshProcessInstance(int processInstanceId) {
         logger.info("process instance update: {}", processInstanceId);
-        processInstance = processService.findProcessInstanceById(processInstanceId);
+        ProcessInstance newProcessInstance = processService.findProcessInstanceById(processInstanceId);
+        // just update the processInstance field(this is soft copy)
+        BeanUtils.copyProperties(newProcessInstance, processInstance);
+
         processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(),
-                                                                 processInstance.getProcessDefinitionVersion());
+            processInstance.getProcessDefinitionVersion());
         processInstance.setProcessDefinition(processDefinition);
     }
 
@@ -773,6 +778,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         }
         // generate process dag
         dag = DagHelper.buildDagGraph(processDag);
+        logger.info("Build dag success, dag: {}", dag);
     }
 
     /**
@@ -787,45 +793,60 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         errorTaskMap.clear();
 
         if (!isNewProcessInstance()) {
+            logger.info("The workflowInstance is not a newly running instance, runtimes: {}, recover flag: {}",
+                processInstance.getRunTimes(),
+                processInstance.getRecovery());
             List<TaskInstance> validTaskInstanceList =
                 processService.findValidTaskListByProcessId(processInstance.getId());
             for (TaskInstance task : validTaskInstanceList) {
-                if (validTaskMap.containsKey(task.getTaskCode())) {
-                    int oldTaskInstanceId = validTaskMap.get(task.getTaskCode());
-                    TaskInstance oldTaskInstance = taskInstanceMap.get(oldTaskInstanceId);
-                    if (!oldTaskInstance.getState().typeIsFinished() && task.getState().typeIsFinished()) {
-                        task.setFlag(Flag.NO);
-                        processService.updateTaskInstance(task);
-                        continue;
+                try {
+                    LoggerUtils.setWorkflowAndTaskInstanceIDMDC(task.getProcessInstanceId(), task.getId());
+                    logger.info(
+                        "Check the taskInstance from a exist workflowInstance, existTaskInstanceCode: {}, taskInstanceStatus: {}",
+                        task.getTaskCode(),
+                        task.getState());
+                    if (validTaskMap.containsKey(task.getTaskCode())) {
+                        int oldTaskInstanceId = validTaskMap.get(task.getTaskCode());
+                        TaskInstance oldTaskInstance = taskInstanceMap.get(oldTaskInstanceId);
+                        if (!oldTaskInstance.getState().typeIsFinished() && task.getState().typeIsFinished()) {
+                            task.setFlag(Flag.NO);
+                            processService.updateTaskInstance(task);
+                            continue;
+                        }
+                        logger.warn("have same taskCode taskInstance when init task queue, taskCode:{}",
+                            task.getTaskCode());
                     }
-                    logger.warn("have same taskCode taskInstance when init task queue, taskCode:{}",
-                        task.getTaskCode());
-                }
 
-                validTaskMap.put(task.getTaskCode(), task.getId());
-                taskInstanceMap.put(task.getId(), task);
+                    validTaskMap.put(task.getTaskCode(), task.getId());
+                    taskInstanceMap.put(task.getId(), task);
 
-                if (task.isTaskComplete()) {
-                    completeTaskMap.put(task.getTaskCode(), task.getId());
-                    continue;
-                }
-                if (task.isConditionsTask() || DagHelper.haveConditionsAfterNode(Long.toString(task.getTaskCode()), dag)) {
-                    continue;
-                }
-                if (task.taskCanRetry()) {
-                    if (task.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE) {
-                        // tolerantTaskInstance add to standby list directly
-                        TaskInstance tolerantTaskInstance = cloneTolerantTaskInstance(task);
-                        addTaskToStandByList(tolerantTaskInstance);
-                    } else {
-                        retryTaskInstance(task);
+                    if (task.isTaskComplete()) {
+                        completeTaskMap.put(task.getTaskCode(), task.getId());
+                        continue;
                     }
-                    continue;
-                }
-                if (task.getState().typeIsFailure()) {
-                    errorTaskMap.put(task.getTaskCode(), task.getId());
+                    if (task.isConditionsTask() || DagHelper.haveConditionsAfterNode(Long.toString(task.getTaskCode()),
+                        dag)) {
+                        continue;
+                    }
+                    if (task.taskCanRetry()) {
+                        if (task.getState() == ExecutionStatus.NEED_FAULT_TOLERANCE) {
+                            // tolerantTaskInstance add to standby list directly
+                            TaskInstance tolerantTaskInstance = cloneTolerantTaskInstance(task);
+                            addTaskToStandByList(tolerantTaskInstance);
+                        } else {
+                            retryTaskInstance(task);
+                        }
+                        continue;
+                    }
+                    if (task.getState().typeIsFailure()) {
+                        errorTaskMap.put(task.getTaskCode(), task.getId());
+                    }
+                } finally {
+                    LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
                 }
             }
+        } else {
+            logger.info("The current workflowInstance is a newly running workflowInstance");
         }
 
         if (processInstance.isComplementData() && complementListDate.isEmpty()) {
@@ -858,6 +879,10 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
                 }
             }
         }
+        logger.info("Initialize task queue, dependFailedTaskMap: {}, completeTaskMap: {}, errorTaskMap: {}",
+            dependFailedTaskMap,
+            completeTaskMap,
+            errorTaskMap);
     }
 
     /**
@@ -902,6 +927,15 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             validTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
             taskInstanceMap.put(taskInstance.getId(), taskInstance);
             activeTaskProcessorMaps.put(taskInstance.getTaskCode(), taskProcessor);
+            boolean dispatchSuccess = taskProcessor.action(TaskAction.DISPATCH);
+            if (!dispatchSuccess) {
+                logger.error("process id:{} name:{} dispatch standby task id:{} name:{} failed!",
+                    processInstance.getId(),
+                    processInstance.getName(),
+                    taskInstance.getId(),
+                    taskInstance.getName());
+                return Optional.empty();
+            }
             taskProcessor.action(TaskAction.RUN);
 
             stateWheelExecuteThread.addTask4TimeoutCheck(processInstance, taskInstance);
@@ -1835,14 +1869,19 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      * is new process instance
      */
     private boolean isNewProcessInstance() {
+        if (Flag.YES.equals(processInstance.getRecovery())) {
+            logger.info("This workInstance will be recover by this execution");
+            return false;
+        }
+
         if (ExecutionStatus.RUNNING_EXECUTION == processInstance.getState() && processInstance.getRunTimes() == 1) {
             return true;
-        } else if (processInstance.getRecovery().equals(Flag.YES)) {
-            // host is empty use old task instance
-            return false;
-        } else {
-            return false;
         }
+        logger.info(
+            "The workflowInstance has been executed before, this execution is to reRun, processInstance status: {}, runTimes: {}",
+            processInstance.getState(),
+            processInstance.getRunTimes());
+        return false;
     }
 
     public Map<Long, Integer> getCompleteTaskMap() {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
index a19d08affc..0dec26defe 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
@@ -67,7 +67,6 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
                 return true;
             }
         }
-        dispatchTask();
         return true;
     }
 
@@ -110,7 +109,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
                 logger.info("submit task, but the status of the task {} is already running or delayed.", taskInstance.getName());
                 return true;
             }
-            logger.info("task ready to submit: taskInstanceId: {}", taskInstance.getId());
+            logger.info("task ready to dispatch to worker: taskInstanceId: {}", taskInstance.getId());
 
             TaskPriority taskPriority = new TaskPriority(processInstance.getProcessInstancePriority().getCode(),
                     processInstance.getId(), taskInstance.getProcessInstancePriority().getCode(),
@@ -158,7 +157,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
             TaskKillRequestCommand killCommand = new TaskKillRequestCommand();
             killCommand.setTaskInstanceId(taskInstance.getId());
 
-            ExecutionContext executionContext = new ExecutionContext(killCommand.convert2Command(), ExecutorType.WORKER);
+            ExecutionContext executionContext = new ExecutionContext(killCommand.convert2Command(), ExecutorType.WORKER, taskInstance);
 
             Host host = Host.of(taskInstance.getHost());
             executionContext.setHost(host);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
index 313a43c0e0..05a54e8529 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
@@ -17,46 +17,12 @@
 
 package org.apache.dolphinscheduler.server.master.service;
 
-import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.server.master.event.StateEvent;
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.common.model.Server;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
-import org.apache.dolphinscheduler.common.utils.NetUtils;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
-import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
-import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
-import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
-import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
-import org.apache.dolphinscheduler.server.utils.ProcessUtils;
-import org.apache.dolphinscheduler.service.process.ProcessService;
-import org.apache.dolphinscheduler.service.registry.RegistryClient;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.time.StopWatch;
-
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
-import io.micrometer.core.annotation.Counted;
-import io.micrometer.core.annotation.Timed;
 import lombok.NonNull;
 
 /**
@@ -65,41 +31,14 @@ import lombok.NonNull;
 @Component
 public class FailoverService {
     private static final Logger LOGGER = LoggerFactory.getLogger(FailoverService.class);
-    private final RegistryClient registryClient;
-    private final MasterConfig masterConfig;
-    private final ProcessService processService;
-    private final WorkflowExecuteThreadPool workflowExecuteThreadPool;
-    private final ProcessInstanceExecCacheManager cacheManager;
-    private final String localAddress;
-
-    public FailoverService(@NonNull RegistryClient registryClient,
-                           @NonNull MasterConfig masterConfig,
-                           @NonNull ProcessService processService,
-                           @NonNull WorkflowExecuteThreadPool workflowExecuteThreadPool,
-                           @NonNull ProcessInstanceExecCacheManager cacheManager) {
-        this.registryClient = registryClient;
-        this.masterConfig = masterConfig;
-        this.processService = processService;
-        this.workflowExecuteThreadPool = workflowExecuteThreadPool;
-        this.cacheManager = cacheManager;
-        this.localAddress = NetUtils.getAddr(masterConfig.getListenPort());
-    }
 
-    /**
-     * check master failover
-     */
-    @Counted(value = "failover_scheduler_check_task_count")
-    @Timed(value = "failover_scheduler_check_task_time", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
-    public void checkMasterFailover() {
-        List<String> hosts = getNeedFailoverMasterServers();
-        if (CollectionUtils.isEmpty(hosts)) {
-            return;
-        }
-        LOGGER.info("Master failover service {} begin to failover hosts:{}", localAddress, hosts);
+    private final MasterFailoverService masterFailoverService;
+    private final WorkerFailoverService workerFailoverService;
 
-        for (String host : hosts) {
-            failoverMasterWithLock(host);
-        }
+    public FailoverService(@NonNull MasterFailoverService masterFailoverService,
+                           @NonNull WorkerFailoverService workerFailoverService) {
+        this.masterFailoverService = masterFailoverService;
+        this.workerFailoverService = workerFailoverService;
     }
 
     /**
@@ -111,304 +50,18 @@ public class FailoverService {
     public void failoverServerWhenDown(String serverHost, NodeType nodeType) {
         switch (nodeType) {
             case MASTER:
-                failoverMasterWithLock(serverHost);
+                LOGGER.info("Master failover starting, masterServer: {}", serverHost);
+                masterFailoverService.failoverMaster(serverHost);
+                LOGGER.info("Master failover finished, masterServer: {}", serverHost);
                 break;
             case WORKER:
-                failoverWorker(serverHost);
+                LOGGER.info("Worker failover staring, workerServer: {}", serverHost);
+                workerFailoverService.failoverWorker(serverHost);
+                LOGGER.info("Worker failover finished, workerServer: {}", serverHost);
                 break;
             default:
                 break;
         }
     }
 
-    private void failoverMasterWithLock(String masterHost) {
-        String failoverPath = getFailoverLockPath(NodeType.MASTER, masterHost);
-        try {
-            registryClient.getLock(failoverPath);
-            this.failoverMaster(masterHost);
-        } catch (Exception e) {
-            LOGGER.error("{} server failover failed, host:{}", NodeType.MASTER, masterHost, e);
-        } finally {
-            registryClient.releaseLock(failoverPath);
-        }
-    }
-
-    /**
-     * Failover master, will failover process instance and associated task instance.
-     * <p>When the process instance belongs to the given masterHost and the restartTime is before the current server start up time,
-     * then the process instance will be failovered.
-     *
-     * @param masterHost master host
-     */
-    private void failoverMaster(String masterHost) {
-        if (StringUtils.isEmpty(masterHost)) {
-            return;
-        }
-        Date serverStartupTime = getServerStartupTime(NodeType.MASTER, masterHost);
-        StopWatch failoverTimeCost = StopWatch.createStarted();
-        List<ProcessInstance> needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost);
-        LOGGER.info("start master[{}] failover, need to failover process list size:{}", masterHost, needFailoverProcessInstanceList.size());
-
-        // servers need to contain master hosts and worker hosts, otherwise the logic task will failover fail.
-        List<Server> servers = registryClient.getServerList(NodeType.WORKER);
-        servers.addAll(registryClient.getServerList(NodeType.MASTER));
-
-        for (ProcessInstance processInstance : needFailoverProcessInstanceList) {
-            if (Constants.NULL.equals(processInstance.getHost())) {
-                continue;
-            }
-
-            List<TaskInstance> validTaskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
-            for (TaskInstance taskInstance : validTaskInstanceList) {
-                LOGGER.info("failover task instance id: {}, process instance id: {}", taskInstance.getId(), taskInstance.getProcessInstanceId());
-                failoverTaskInstance(processInstance, taskInstance, servers);
-            }
-
-            if (serverStartupTime != null && processInstance.getRestartTime() != null
-                    && processInstance.getRestartTime().after(serverStartupTime)) {
-                continue;
-            }
-
-            LOGGER.info("failover process instance id: {}", processInstance.getId());
-            ProcessInstanceMetrics.incProcessInstanceFailover();
-            //updateProcessInstance host is null and insert into command
-            processInstance.setHost(Constants.NULL);
-            processService.processNeedFailoverProcessInstances(processInstance);
-        }
-
-        failoverTimeCost.stop();
-        LOGGER.info("master[{}] failover end, useTime:{}ms", masterHost, failoverTimeCost.getTime(TimeUnit.MILLISECONDS));
-    }
-
-    /**
-     * Do the worker failover. Will find the SUBMITTED_SUCCESS/DISPATCH/RUNNING_EXECUTION/DELAY_EXECUTION/READY_PAUSE/READY_STOP tasks belong the given worker,
-     * and failover these tasks.
-     * <p>
-     * Note: When we do worker failover, the master will only failover the processInstance belongs to the current master.
-     *
-     * @param workerHost worker host
-     */
-    private void failoverWorker(String workerHost) {
-        if (StringUtils.isEmpty(workerHost)) {
-            return;
-        }
-
-        long startTime = System.currentTimeMillis();
-        // we query the task instance from cache, so that we can directly update the cache
-        final List<TaskInstance> needFailoverTaskInstanceList = cacheManager.getAll()
-            .stream()
-            .flatMap(workflowExecuteRunnable -> workflowExecuteRunnable.getAllTaskInstances().stream())
-            .filter(taskInstance ->
-                workerHost.equals(taskInstance.getHost()) && ExecutionStatus.isNeedFailoverWorkflowInstanceState(taskInstance.getState()))
-            .collect(Collectors.toList());
-        final Map<Integer, ProcessInstance> processInstanceCacheMap = new HashMap<>();
-        LOGGER.info("start worker[{}] failover, task list size:{}", workerHost, needFailoverTaskInstanceList.size());
-        final List<Server> workerServers = registryClient.getServerList(NodeType.WORKER);
-        for (TaskInstance taskInstance : needFailoverTaskInstanceList) {
-            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskInstance.getProcessInstanceId(), taskInstance.getId());
-            try {
-                ProcessInstance processInstance = processInstanceCacheMap.get(taskInstance.getProcessInstanceId());
-                if (processInstance == null) {
-                    processInstance = cacheManager.getByProcessInstanceId(taskInstance.getProcessInstanceId()).getProcessInstance();
-                    if (processInstance == null) {
-                        LOGGER.error("failover task instance error, processInstance {} of taskInstance {} is null",
-                            taskInstance.getProcessInstanceId(), taskInstance.getId());
-                        continue;
-                    }
-                    processInstanceCacheMap.put(processInstance.getId(), processInstance);
-                }
-
-                // only failover the task owned myself if worker down.
-                if (!StringUtils.equalsIgnoreCase(processInstance.getHost(), localAddress)) {
-                    continue;
-                }
-
-                LOGGER.info("failover task instance id: {}, process instance id: {}", taskInstance.getId(), taskInstance.getProcessInstanceId());
-                failoverTaskInstance(processInstance, taskInstance, workerServers);
-            } finally {
-                LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
-            }
-        }
-        LOGGER.info("end worker[{}] failover, useTime:{}ms", workerHost, System.currentTimeMillis() - startTime);
-    }
-
-    /**
-     * failover task instance
-     * <p>
-     * 1. kill yarn job if run on worker and there are yarn jobs in tasks.
-     * 2. change task state from running to need failover.
-     * 3. try to notify local master
-     *
-     * @param processInstance
-     * @param taskInstance
-     * @param servers         if failover master, servers container master servers and worker servers; if failover worker, servers contain worker servers.
-     */
-    private void failoverTaskInstance(@NonNull ProcessInstance processInstance, TaskInstance taskInstance, List<Server> servers) {
-        if (!checkTaskInstanceNeedFailover(servers, taskInstance)) {
-            LOGGER.info("The taskInstance doesn't need to failover");
-            return;
-        }
-        TaskMetrics.incTaskFailover();
-        boolean isMasterTask = TaskProcessorFactory.isMasterTask(taskInstance.getTaskType());
-
-        taskInstance.setProcessInstance(processInstance);
-
-        if (!isMasterTask) {
-            LOGGER.info("The failover taskInstance is not master task");
-            TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
-                .buildTaskInstanceRelatedInfo(taskInstance)
-                .buildProcessInstanceRelatedInfo(processInstance)
-                .create();
-
-            if (masterConfig.isKillYarnJobWhenTaskFailover()) {
-                // only kill yarn job if exists , the local thread has exited
-                ProcessUtils.killYarnJob(taskExecutionContext);
-            }
-        } else {
-            LOGGER.info("The failover taskInstance is a master task");
-        }
-
-        taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE);
-        processService.saveTaskInstance(taskInstance);
-
-        StateEvent stateEvent = new StateEvent();
-        stateEvent.setTaskInstanceId(taskInstance.getId());
-        stateEvent.setType(StateEventType.TASK_STATE_CHANGE);
-        stateEvent.setProcessInstanceId(processInstance.getId());
-        stateEvent.setExecutionStatus(taskInstance.getState());
-        workflowExecuteThreadPool.submitStateEvent(stateEvent);
-    }
-
-    /**
-     * Get need failover master servers.
-     * <p>
-     * Query the process instances from database, if the processInstance's host doesn't exist in registry
-     * or the host is the currentServer, then it will need to failover.
-     *
-     * @return need failover master servers
-     */
-    private List<String> getNeedFailoverMasterServers() {
-        // failover myself && failover dead masters
-        List<String> hosts = processService.queryNeedFailoverProcessInstanceHost();
-
-        Iterator<String> iterator = hosts.iterator();
-        while (iterator.hasNext()) {
-            String host = iterator.next();
-            if (registryClient.checkNodeExists(host, NodeType.MASTER)) {
-                if (!localAddress.equals(host)) {
-                    iterator.remove();
-                }
-            }
-        }
-        return hosts;
-    }
-
-    /**
-     * task needs failover if task start before server starts
-     *
-     * @param servers servers, can container master servers or worker servers
-     * @param taskInstance  task instance
-     * @return true if task instance need fail over
-     */
-    private boolean checkTaskInstanceNeedFailover(List<Server> servers, TaskInstance taskInstance) {
-
-        boolean taskNeedFailover = true;
-
-        if (taskInstance == null) {
-            LOGGER.error("Master failover task instance error, taskInstance is null");
-            return false;
-        }
-
-        if (Constants.NULL.equals(taskInstance.getHost())) {
-            return false;
-        }
-
-        if (taskInstance.getState() != null && taskInstance.getState().typeIsFinished()) {
-            return false;
-        }
-
-        //now no host will execute this task instance,so no need to failover the task
-        if (taskInstance.getHost() == null) {
-            return false;
-        }
-
-        //if task start after server starts, there is no need to failover the task.
-        if (checkTaskAfterServerStart(servers, taskInstance)) {
-            taskNeedFailover = false;
-        }
-
-        return taskNeedFailover;
-    }
-
-    /**
-     * check task start after the worker server starts.
-     *
-     * @param servers servers, can contain master servers or worker servers
-     * @param taskInstance task instance
-     * @return true if task instance start time after server start date
-     */
-    private boolean checkTaskAfterServerStart(List<Server> servers, TaskInstance taskInstance) {
-        if (StringUtils.isEmpty(taskInstance.getHost())) {
-            return false;
-        }
-        Date serverStartDate = getServerStartupTime(servers, taskInstance.getHost());
-        if (serverStartDate != null) {
-            if (taskInstance.getStartTime() == null) {
-                return taskInstance.getSubmitTime().after(serverStartDate);
-            } else {
-                return taskInstance.getStartTime().after(serverStartDate);
-            }
-        }
-        return false;
-    }
-
-    /**
-     * get failover lock path
-     *
-     * @param nodeType zookeeper node type
-     * @return fail over lock path
-     */
-    private String getFailoverLockPath(NodeType nodeType, String host) {
-        switch (nodeType) {
-            case MASTER:
-                return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS + "/" + host;
-            case WORKER:
-                return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS + "/" + host;
-            default:
-                return "";
-        }
-    }
-
-    /**
-     * get server startup time
-     */
-    private Date getServerStartupTime(NodeType nodeType, String host) {
-        if (StringUtils.isEmpty(host)) {
-            return null;
-        }
-        List<Server> servers = registryClient.getServerList(nodeType);
-        return getServerStartupTime(servers, host);
-    }
-
-    /**
-     * get server startup time
-     */
-    private Date getServerStartupTime(List<Server> servers, String host) {
-        if (CollectionUtils.isEmpty(servers)) {
-            return null;
-        }
-        Date serverStartupTime = null;
-        for (Server server : servers) {
-            if (host.equals(server.getHost() + Constants.COLON + server.getPort())) {
-                serverStartupTime = server.getCreateTime();
-                break;
-            }
-        }
-        return serverStartupTime;
-    }
-
-    public String getLocalAddress() {
-        return localAddress;
-    }
-
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
new file mode 100644
index 0000000000..c7e5b4ea13
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.service;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.Flag;
+import org.apache.dolphinscheduler.common.enums.NodeType;
+import org.apache.dolphinscheduler.common.model.Server;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
+import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
+import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
+import org.apache.dolphinscheduler.server.utils.ProcessUtils;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.registry.RegistryClient;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.time.StopWatch;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import io.micrometer.core.annotation.Counted;
+import io.micrometer.core.annotation.Timed;
+import lombok.NonNull;
+
+@Service
+public class MasterFailoverService {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(MasterFailoverService.class);
+    private final RegistryClient registryClient;
+    private final MasterConfig masterConfig;
+    private final ProcessService processService;
+    private final String localAddress;
+
+    public MasterFailoverService(@NonNull RegistryClient registryClient,
+                                 @NonNull MasterConfig masterConfig,
+                                 @NonNull ProcessService processService) {
+        this.registryClient = registryClient;
+        this.masterConfig = masterConfig;
+        this.processService = processService;
+        this.localAddress = NetUtils.getAddr(masterConfig.getListenPort());
+
+    }
+
+    /**
+     * check master failover
+     */
+    @Counted(value = "ds.master.scheduler.failover.check.count")
+    @Timed(value = "ds.master.scheduler.failover.check.time", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
+    public void checkMasterFailover() {
+        List<String> needFailoverMasterHosts = processService.queryNeedFailoverProcessInstanceHost()
+            .stream()
+            // failover myself || dead server
+            .filter(host -> localAddress.equals(host) || !registryClient.checkNodeExists(host, NodeType.MASTER))
+            .distinct()
+            .collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(needFailoverMasterHosts)) {
+            return;
+        }
+        LOGGER.info("Master failover service {} begin to failover hosts:{}", localAddress, needFailoverMasterHosts);
+
+        for (String needFailoverMasterHost : needFailoverMasterHosts) {
+            failoverMaster(needFailoverMasterHost);
+        }
+    }
+
+    public void failoverMaster(String masterHost) {
+        String failoverPath = Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS + "/" + masterHost;
+        try {
+            registryClient.getLock(failoverPath);
+            doFailoverMaster(masterHost);
+        } catch (Exception e) {
+            LOGGER.error("Master server failover failed, host:{}", masterHost, e);
+        } finally {
+            registryClient.releaseLock(failoverPath);
+        }
+    }
+
+    /**
+     * Failover master, will failover process instance and associated task instance.
+     * <p>When the process instance belongs to the given masterHost and the restartTime is before the current server start up time,
+     * then the process instance will be failovered.
+     *
+     * @param masterHost master host
+     */
+    private void doFailoverMaster(@NonNull String masterHost) {
+        LOGGER.info("Master[{}] failover starting, need to failover process", masterHost);
+        StopWatch failoverTimeCost = StopWatch.createStarted();
+
+        Optional<Date> masterStartupTimeOptional =
+            getServerStartupTime(registryClient.getServerList(NodeType.MASTER), masterHost);
+        List<ProcessInstance> needFailoverProcessInstanceList =
+            processService.queryNeedFailoverProcessInstances(masterHost);
+
+        LOGGER.info(
+            "Master[{}] failover there are {} workflowInstance may need to failover, will do a deep check, workflowInstanceIds: {}",
+            masterHost,
+            needFailoverProcessInstanceList.size(),
+            needFailoverProcessInstanceList.stream().map(ProcessInstance::getId).collect(Collectors.toList()));
+
+        for (ProcessInstance processInstance : needFailoverProcessInstanceList) {
+            try {
+                LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
+                LOGGER.info("WorkflowInstance failover starting");
+                if (!checkProcessInstanceNeedFailover(masterStartupTimeOptional, processInstance)) {
+                    LOGGER.info("WorkflowInstance doesn't need to failover");
+                    continue;
+                }
+                int processInstanceId = processInstance.getId();
+                List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstanceId);
+                for (TaskInstance taskInstance : taskInstanceList) {
+                    try {
+                        LoggerUtils.setTaskInstanceIdMDC(taskInstance.getId());
+                        LOGGER.info("TaskInstance failover starting");
+                        if (!checkTaskInstanceNeedFailover(taskInstance)) {
+                            LOGGER.info("The taskInstance doesn't need to failover");
+                            continue;
+                        }
+                        failoverTaskInstance(processInstance, taskInstance);
+                        LOGGER.info("TaskInstance failover finished");
+                    } finally {
+                        LoggerUtils.removeTaskInstanceIdMDC();
+                    }
+                }
+
+                ProcessInstanceMetrics.incProcessInstanceFailover();
+                //updateProcessInstance host is null to mark this processInstance has been failover
+                // and insert a failover command
+                processInstance.setHost(Constants.NULL);
+                processService.processNeedFailoverProcessInstances(processInstance);
+                LOGGER.info("WorkflowInstance failover finished");
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
+            }
+        }
+
+        failoverTimeCost.stop();
+        LOGGER.info("Master[{}] failover finished, useTime:{}ms",
+            masterHost,
+            failoverTimeCost.getTime(TimeUnit.MILLISECONDS));
+    }
+
+    private Optional<Date> getServerStartupTime(List<Server> servers, String host) {
+        if (CollectionUtils.isEmpty(servers)) {
+            return Optional.empty();
+        }
+        Date serverStartupTime = null;
+        for (Server server : servers) {
+            if (host.equals(server.getHost() + Constants.COLON + server.getPort())) {
+                serverStartupTime = server.getCreateTime();
+                break;
+            }
+        }
+        return Optional.ofNullable(serverStartupTime);
+    }
+
+    /**
+     * failover task instance
+     * <p>
+     * 1. kill yarn job if run on worker and there are yarn jobs in tasks.
+     * 2. change task state from running to need failover.
+     * 3. try to notify local master
+     *
+     * @param processInstance
+     * @param taskInstance
+     */
+    private void failoverTaskInstance(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
+        TaskMetrics.incTaskFailover();
+        boolean isMasterTask = TaskProcessorFactory.isMasterTask(taskInstance.getTaskType());
+
+        taskInstance.setProcessInstance(processInstance);
+
+        if (!isMasterTask) {
+            LOGGER.info("The failover taskInstance is not master task");
+            TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
+                .buildTaskInstanceRelatedInfo(taskInstance)
+                .buildProcessInstanceRelatedInfo(processInstance)
+                .create();
+
+            if (masterConfig.isKillYarnJobWhenTaskFailover()) {
+                // only kill yarn job if exists , the local thread has exited
+                LOGGER.info("TaskInstance failover begin kill the task related yarn job");
+                ProcessUtils.killYarnJob(taskExecutionContext);
+            }
+        } else {
+            LOGGER.info("The failover taskInstance is a master task");
+        }
+
+        taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE);
+        taskInstance.setFlag(Flag.NO);
+        processService.saveTaskInstance(taskInstance);
+    }
+
+    private boolean checkTaskInstanceNeedFailover(@NonNull TaskInstance taskInstance) {
+        if (taskInstance.getState() != null && taskInstance.getState().typeIsFinished()) {
+            // The task is already finished, so we don't need to failover this task instance
+            return false;
+        }
+        return true;
+    }
+
+    private boolean checkProcessInstanceNeedFailover(Optional<Date> beFailoveredMasterStartupTimeOptional,
+                                                     @NonNull ProcessInstance processInstance) {
+        // The process has already been failover, since when we do master failover we will hold a lock, so we can guarantee
+        // the host will not be set concurrent.
+        if (Constants.NULL.equals(processInstance.getHost())) {
+            return false;
+        }
+        if (!beFailoveredMasterStartupTimeOptional.isPresent()) {
+            // the master is not active, we can failover all it's processInstance
+            return true;
+        }
+        Date beFailoveredMasterStartupTime = beFailoveredMasterStartupTimeOptional.get();
+
+        if (processInstance.getStartTime().after(beFailoveredMasterStartupTime)) {
+            // The processInstance is newly created
+            return false;
+        }
+
+        return true;
+    }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java
new file mode 100644
index 0000000000..ec126a3ec3
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.service;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.Flag;
+import org.apache.dolphinscheduler.common.enums.NodeType;
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.model.Server;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
+import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
+import org.apache.dolphinscheduler.server.utils.ProcessUtils;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.registry.RegistryClient;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.StopWatch;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import lombok.NonNull;
+
+@Service
+public class WorkerFailoverService {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(WorkerFailoverService.class);
+
+    private final RegistryClient registryClient;
+    private final MasterConfig masterConfig;
+    private final ProcessService processService;
+    private final WorkflowExecuteThreadPool workflowExecuteThreadPool;
+    private final ProcessInstanceExecCacheManager cacheManager;
+    private final String localAddress;
+
+    public WorkerFailoverService(@NonNull RegistryClient registryClient,
+                                 @NonNull MasterConfig masterConfig,
+                                 @NonNull ProcessService processService,
+                                 @NonNull WorkflowExecuteThreadPool workflowExecuteThreadPool,
+                                 @NonNull ProcessInstanceExecCacheManager cacheManager) {
+        this.registryClient = registryClient;
+        this.masterConfig = masterConfig;
+        this.processService = processService;
+        this.workflowExecuteThreadPool = workflowExecuteThreadPool;
+        this.cacheManager = cacheManager;
+        this.localAddress = NetUtils.getAddr(masterConfig.getListenPort());
+    }
+
+    /**
+     * Do the worker failover. Will find the SUBMITTED_SUCCESS/DISPATCH/RUNNING_EXECUTION/DELAY_EXECUTION/READY_PAUSE/READY_STOP tasks belong the given worker,
+     * and failover these tasks.
+     * <p>
+     * Note: When we do worker failover, the master will only failover the processInstance belongs to the current master.
+     *
+     * @param workerHost worker host
+     */
+    public void failoverWorker(@NonNull String workerHost) {
+        LOGGER.info("Worker[{}] failover starting", workerHost);
+        final StopWatch failoverTimeCost = StopWatch.createStarted();
+
+        // we query the task instance from cache, so that we can directly update the cache
+        final Optional<Date> needFailoverWorkerStartTime =
+            getServerStartupTime(registryClient.getServerList(NodeType.WORKER), workerHost);
+
+        final List<TaskInstance> needFailoverTaskInstanceList = getNeedFailoverTaskInstance(workerHost);
+        if (CollectionUtils.isEmpty(needFailoverTaskInstanceList)) {
+            LOGGER.info("Worker[{}] failover finished there are no taskInstance need to failover", workerHost);
+            return;
+        }
+        LOGGER.info(
+            "Worker[{}] failover there are {} taskInstance may need to failover, will do a deep check, taskInstanceIds: {}",
+            workerHost,
+            needFailoverTaskInstanceList.size(),
+            needFailoverTaskInstanceList.stream().map(TaskInstance::getId).collect(Collectors.toList()));
+        final Map<Integer, ProcessInstance> processInstanceCacheMap = new HashMap<>();
+        for (TaskInstance taskInstance : needFailoverTaskInstanceList) {
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskInstance.getProcessInstanceId(), taskInstance.getId());
+            try {
+                ProcessInstance processInstance =
+                    processInstanceCacheMap.computeIfAbsent(taskInstance.getProcessInstanceId(), k -> {
+                        WorkflowExecuteRunnable workflowExecuteRunnable =
+                            cacheManager.getByProcessInstanceId(taskInstance.getProcessInstanceId());
+                        if (workflowExecuteRunnable == null) {
+                            return null;
+                        }
+                        return workflowExecuteRunnable.getProcessInstance();
+                    });
+                if (!checkTaskInstanceNeedFailover(needFailoverWorkerStartTime, processInstance, taskInstance)) {
+                    LOGGER.info("Worker[{}] the current taskInstance doesn't need to failover", workerHost);
+                    continue;
+                }
+                LOGGER.info(
+                    "Worker[{}] failover: begin to failover taskInstance, will set the status to NEED_FAULT_TOLERANCE",
+                    workerHost);
+                failoverTaskInstance(processInstance, taskInstance);
+                LOGGER.info("Worker[{}] failover: Finish failover taskInstance", workerHost);
+            } catch (Exception ex) {
+                LOGGER.info("Worker[{}] failover taskInstance occur exception", workerHost, ex);
+            } finally {
+                LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
+            }
+        }
+        failoverTimeCost.stop();
+        LOGGER.info("Worker[{}] failover finished, useTime:{}ms",
+            workerHost,
+            failoverTimeCost.getTime(TimeUnit.MILLISECONDS));
+    }
+
+    /**
+     * failover task instance
+     * <p>
+     * 1. kill yarn job if run on worker and there are yarn jobs in tasks.
+     * 2. change task state from running to need failover.
+     * 3. try to notify local master
+     *
+     * @param processInstance
+     * @param taskInstance
+     */
+    private void failoverTaskInstance(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
+
+        TaskMetrics.incTaskFailover();
+        boolean isMasterTask = TaskProcessorFactory.isMasterTask(taskInstance.getTaskType());
+
+        taskInstance.setProcessInstance(processInstance);
+
+        if (!isMasterTask) {
+            LOGGER.info("The failover taskInstance is not master task");
+            TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
+                .buildTaskInstanceRelatedInfo(taskInstance)
+                .buildProcessInstanceRelatedInfo(processInstance)
+                .create();
+
+            if (masterConfig.isKillYarnJobWhenTaskFailover()) {
+                // only kill yarn job if exists , the local thread has exited
+                LOGGER.info("TaskInstance failover begin kill the task related yarn job");
+                ProcessUtils.killYarnJob(taskExecutionContext);
+            }
+        } else {
+            LOGGER.info("The failover taskInstance is a master task");
+        }
+
+        taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE);
+        taskInstance.setFlag(Flag.NO);
+        processService.saveTaskInstance(taskInstance);
+
+        StateEvent stateEvent = new StateEvent();
+        stateEvent.setTaskInstanceId(taskInstance.getId());
+        stateEvent.setType(StateEventType.TASK_STATE_CHANGE);
+        stateEvent.setProcessInstanceId(processInstance.getId());
+        stateEvent.setExecutionStatus(taskInstance.getState());
+        workflowExecuteThreadPool.submitStateEvent(stateEvent);
+    }
+
+    /**
+     * task needs failover if task start before server starts
+     *
+     * @return true if task instance need fail over
+     */
+    private boolean checkTaskInstanceNeedFailover(Optional<Date> needFailoverWorkerStartTime,
+                                                  @Nullable ProcessInstance processInstance,
+                                                  TaskInstance taskInstance) {
+        if (processInstance == null) {
+            // This case should be happened.
+            LOGGER.error(
+                "Failover task instance error, cannot find the related processInstance form memory, this case shouldn't happened");
+            return false;
+        }
+        if (taskInstance == null) {
+            // This case should be happened.
+            LOGGER.error("Master failover task instance error, taskInstance is null, this case shouldn't happened");
+            return false;
+        }
+        // only failover the task owned myself if worker down.
+        if (!StringUtils.equalsIgnoreCase(processInstance.getHost(), localAddress)) {
+            LOGGER.error(
+                "Master failover task instance error, the taskInstance's processInstance's host: {} is not the current master: {}",
+                processInstance.getHost(),
+                localAddress);
+            return false;
+        }
+        if (taskInstance.getState() != null && taskInstance.getState().typeIsFinished()) {
+            // The taskInstance is already finished, doesn't need to failover
+            LOGGER.info("The task is already finished, doesn't need to failover");
+            return false;
+        }
+        if (!needFailoverWorkerStartTime.isPresent()) {
+            // The worker is still down
+            return true;
+        }
+        // The worker is active, may already send some new task to it
+        if (taskInstance.getSubmitTime() != null && taskInstance.getSubmitTime()
+            .after(needFailoverWorkerStartTime.get())) {
+            LOGGER.info(
+                "The taskInstance's submitTime: {} is after the need failover worker's start time: {}, the taskInstance is newly submit, it doesn't need to failover",
+                taskInstance.getSubmitTime(),
+                needFailoverWorkerStartTime.get());
+            return false;
+        }
+
+        return true;
+    }
+
+    private List<TaskInstance> getNeedFailoverTaskInstance(@NonNull String failoverWorkerHost) {
+        // we query the task instance from cache, so that we can directly update the cache
+        return cacheManager.getAll()
+            .stream()
+            .flatMap(workflowExecuteRunnable -> workflowExecuteRunnable.getAllTaskInstances().stream())
+            // If the worker is in dispatching and the host is not set
+            .filter(taskInstance -> failoverWorkerHost.equals(taskInstance.getHost())
+                && ExecutionStatus.isNeedFailoverWorkflowInstanceState(taskInstance.getState()))
+            .collect(Collectors.toList());
+    }
+
+    private Optional<Date> getServerStartupTime(List<Server> servers, String host) {
+        if (CollectionUtils.isEmpty(servers)) {
+            return Optional.empty();
+        }
+        Date serverStartupTime = null;
+        for (Server server : servers) {
+            if (host.equals(server.getHost() + Constants.COLON + server.getPort())) {
+                serverStartupTime = server.getCreateTime();
+                break;
+            }
+        }
+        return Optional.ofNullable(serverStartupTime);
+    }
+}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
index b2c7d63cd7..55dd236f71 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
@@ -52,7 +52,7 @@ public class ExecutionContextTestUtils {
         TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(context);
         Command command = requestCommand.convert2Command();
 
-        ExecutionContext executionContext = new ExecutionContext(command, ExecutorType.WORKER);
+        ExecutionContext executionContext = new ExecutionContext(command, ExecutorType.WORKER, taskInstance);
         executionContext.setHost(Host.of(NetUtils.getAddr(port)));
 
         return executionContext;
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
index fdd79552b1..1ee890faf2 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
@@ -70,7 +70,7 @@ public class NettyExecutorManagerTest {
                 .buildProcessInstanceRelatedInfo(processInstance)
                 .buildProcessDefinitionRelatedInfo(processDefinition)
                 .create();
-        ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER);
+        ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, taskInstance);
         executionContext.setHost(Host.of(NetUtils.getAddr(serverConfig.getListenPort())));
         Boolean execute = nettyExecutorManager.execute(executionContext);
         Assert.assertTrue(execute);
@@ -89,7 +89,7 @@ public class NettyExecutorManagerTest {
                 .buildProcessInstanceRelatedInfo(processInstance)
                 .buildProcessDefinitionRelatedInfo(processDefinition)
                 .create();
-        ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER);
+        ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, taskInstance);
         executionContext.setHost(Host.of(NetUtils.getAddr(4444)));
         nettyExecutorManager.execute(executionContext);
 
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
index 98bf514730..44e5a382f6 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
@@ -26,13 +26,14 @@ import static org.mockito.Mockito.doNothing;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.model.Server;
+import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
@@ -98,9 +99,17 @@ public class FailoverServiceTest {
         springApplicationContext.setApplicationContext(applicationContext);
 
         given(masterConfig.getListenPort()).willReturn(masterPort);
-        failoverService = new FailoverService(registryClient, masterConfig, processService, workflowExecuteThreadPool, cacheManager);
+        MasterFailoverService masterFailoverService =
+            new MasterFailoverService(registryClient, masterConfig, processService);
+        WorkerFailoverService workerFailoverService = new WorkerFailoverService(registryClient,
+            masterConfig,
+            processService,
+            workflowExecuteThreadPool,
+            cacheManager);
+
+        failoverService = new FailoverService(masterFailoverService, workerFailoverService);
 
-        testMasterHost = failoverService.getLocalAddress();
+        testMasterHost = NetUtils.getAddr(masterConfig.getListenPort());
         String ip = testMasterHost.split(":")[0];
         int port = Integer.valueOf(testMasterHost.split(":")[1]);
         Assert.assertEquals(masterPort, port);
@@ -118,6 +127,7 @@ public class FailoverServiceTest {
         processInstance = new ProcessInstance();
         processInstance.setId(1);
         processInstance.setHost(testMasterHost);
+        processInstance.setStartTime(new Date());
         processInstance.setRestartTime(new Date());
         processInstance.setHistoryCmd("xxx");
         processInstance.setCommandType(CommandType.STOP);
@@ -154,16 +164,10 @@ public class FailoverServiceTest {
 
         given(registryClient.getServerList(NodeType.WORKER)).willReturn(new ArrayList<>(Arrays.asList(workerServer)));
         given(registryClient.getServerList(NodeType.MASTER)).willReturn(new ArrayList<>(Arrays.asList(masterServer)));
-        ReflectionTestUtils.setField(failoverService, "registryClient", registryClient);
 
         doNothing().when(workflowExecuteThreadPool).submitStateEvent(Mockito.any(StateEvent.class));
     }
 
-    @Test
-    public void checkMasterFailoverTest() {
-        failoverService.checkMasterFailover();
-    }
-
     @Test
     public void failoverMasterTest() {
         processInstance.setHost(Constants.NULL);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index 4bfb0b4809..f61676e693 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
 import org.apache.dolphinscheduler.common.graph.DAG;
 import org.apache.dolphinscheduler.common.model.TaskNode;
 import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
+import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
 import org.apache.dolphinscheduler.dao.entity.Command;
 import org.apache.dolphinscheduler.dao.entity.DagData;
 import org.apache.dolphinscheduler.dao.entity.DataSource;
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index c527276933..5624f47812 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -126,7 +126,6 @@ import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
 import org.apache.dolphinscheduler.remote.command.TaskEventChangeCommand;
 import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
 import org.apache.dolphinscheduler.remote.utils.Host;
-import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.exceptions.ServiceException;
 import org.apache.dolphinscheduler.service.log.LogClientService;
 import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
@@ -266,10 +265,12 @@ public class ProcessServiceImpl implements ProcessService {
     @Autowired
     private TaskPluginManager taskPluginManager;
 
+    @Autowired
+    private ProcessService processService;
+
     /**
      * handle Command (construct ProcessInstance from Command) , wrapped in transaction
      *
-     * @param logger  logger
      * @param host    host
      * @param command found command
      * @return process instance
@@ -904,7 +905,8 @@ public class ProcessServiceImpl implements ProcessService {
         ProcessDefinition processDefinition;
         CommandType commandType = command.getCommandType();
 
-        processDefinition = this.findProcessDefinition(command.getProcessDefinitionCode(), command.getProcessDefinitionVersion());
+        processDefinition = this.findProcessDefinition(command.getProcessDefinitionCode(),
+                                                       command.getProcessDefinitionVersion());
         if (processDefinition == null) {
             logger.error("cannot find the work process define! define code : {}", command.getProcessDefinitionCode());
             return null;
@@ -1003,6 +1005,7 @@ public class ProcessServiceImpl implements ProcessService {
             case RECOVER_TOLERANCE_FAULT_PROCESS:
                 // recover tolerance fault process
                 processInstance.setRecovery(Flag.YES);
+                processInstance.setRunTimes(runTime + 1);
                 runStatus = processInstance.getState();
                 break;
             case COMPLEMENT_DATA:
@@ -1241,11 +1244,15 @@ public class ProcessServiceImpl implements ProcessService {
         while (retryTimes <= commitRetryTimes) {
             try {
                 // submit task to db
-                task = SpringApplicationContext.getBean(ProcessService.class).submitTask(processInstance, taskInstance);
+                // Only want to use transaction here
+                task = processService.submitTask(processInstance, taskInstance);
                 if (task != null && task.getId() != 0) {
                     break;
                 }
-                logger.error("task commit to db failed , taskId {} has already retry {} times, please check the database", taskInstance.getId(), retryTimes);
+                logger.error(
+                    "task commit to db failed , taskId {} has already retry {} times, please check the database",
+                    taskInstance.getId(),
+                    retryTimes);
                 Thread.sleep(commitInterval);
             } catch (Exception e) {
                 logger.error("task commit to db failed", e);
@@ -1267,13 +1274,17 @@ public class ProcessServiceImpl implements ProcessService {
     @Override
     @Transactional(rollbackFor = Exception.class)
     public TaskInstance submitTask(ProcessInstance processInstance, TaskInstance taskInstance) {
-        logger.info("start submit task : {}, processInstance id:{}, state: {}",
-            taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState());
+        logger.info("Start save taskInstance to database : {}, processInstance id:{}, state: {}",
+                    taskInstance.getName(),
+                    taskInstance.getProcessInstanceId(),
+                    processInstance.getState());
         //submit to db
         TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance);
         if (task == null) {
-            logger.error("end submit task to db error, task name:{}, process id:{} state: {} ",
-                taskInstance.getName(), taskInstance.getProcessInstance(), processInstance.getState());
+            logger.error("Save taskInstance to db error, task name:{}, process id:{} state: {} ",
+                         taskInstance.getName(),
+                         taskInstance.getProcessInstance(),
+                         processInstance.getState());
             return null;
         }
 
@@ -1281,8 +1292,13 @@ public class ProcessServiceImpl implements ProcessService {
             createSubWorkProcess(processInstance, task);
         }
 
-        logger.info("end submit task to db successfully:{} {} state:{} complete, instance id:{} state: {}  ",
-            taskInstance.getId(), taskInstance.getName(), task.getState(), processInstance.getId(), processInstance.getState());
+        logger.info(
+            "End save taskInstance to db successfully:{}, taskInstanceName: {}, taskInstance state:{}, processInstanceId:{}, processInstanceState: {}",
+            taskInstance.getId(),
+            taskInstance.getName(),
+            task.getState(),
+            processInstance.getId(),
+            processInstance.getState());
         return task;
     }
 
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java
index 0aeec4609d..823ec81d3c 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriority.java
@@ -211,16 +211,45 @@ public class TaskPriority implements Comparable<TaskPriority> {
         }
         TaskPriority that = (TaskPriority) o;
         return processInstancePriority == that.processInstancePriority
-                && processInstanceId == that.processInstanceId
-                && taskInstancePriority == that.taskInstancePriority
-                && taskId == that.taskId
-                && taskGroupPriority == that.taskGroupPriority
-                && Objects.equals(groupName, that.groupName);
+            && processInstanceId == that.processInstanceId
+            && taskInstancePriority == that.taskInstancePriority
+            && taskId == that.taskId
+            && taskGroupPriority == that.taskGroupPriority
+            && Objects.equals(groupName, that.groupName);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(processInstancePriority, processInstanceId, taskInstancePriority, taskId, taskGroupPriority, groupName);
+        return Objects.hash(processInstancePriority,
+            processInstanceId,
+            taskInstancePriority,
+            taskId,
+            taskGroupPriority,
+            groupName);
     }
 
+    @Override
+    public String toString() {
+        return "TaskPriority{"
+            + "processInstancePriority="
+            + processInstancePriority
+            + ", processInstanceId="
+            + processInstanceId
+            + ", taskInstancePriority="
+            + taskInstancePriority
+            + ", taskId="
+            + taskId
+            + ", taskExecutionContext="
+            + taskExecutionContext
+            + ", groupName='"
+            + groupName
+            + '\''
+            + ", context="
+            + context
+            + ", checkpoint="
+            + checkpoint
+            + ", taskGroupPriority="
+            + taskGroupPriority
+            + '}';
+    }
 }


[dolphinscheduler] 02/29: Optimize MasterServer, add MasterRPCService (#10371)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 81cadd15d2b7058eb94f84ad29d25f71324d4253
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Thu Jun 9 10:00:05 2022 +0800

    Optimize MasterServer, add MasterRPCService (#10371)
    
    * Optimize MasterServer, avoid NPE
    
    (cherry picked from commit 3ecbee3885090b987e120667e1ccf6f6c24d1df8)
---
 .../apache/dolphinscheduler/common/Constants.java  |   2 +-
 .../server/master/MasterServer.java                |  66 ++---------
 .../cache/ProcessInstanceExecCacheManager.java     |   8 +-
 .../impl/ProcessInstanceExecCacheManagerImpl.java  |  10 +-
 .../processor/queue/StateEventResponseService.java |   4 +-
 ...ExecuteThread.java => TaskExecuteRunnable.java} |  37 +++---
 .../processor/queue/TaskExecuteThreadPool.java     |  16 +--
 .../registry/MasterConnectionStateListener.java    |  62 ++++++++++
 .../master/registry/MasterRegistryClient.java      |  84 ++++---------
 .../MasterRPCServer.java}                          | 130 ++++-----------------
 .../server/master/runner/EventExecuteService.java  |   2 +-
 .../master/runner/FailoverExecuteThread.java       |   2 +
 .../master/runner/MasterSchedulerService.java      |   2 +-
 .../master/runner/StateWheelExecuteThread.java     |  24 ++--
 ...uteThread.java => WorkflowExecuteRunnable.java} | 100 ++++++++--------
 .../master/runner/WorkflowExecuteThreadPool.java   |  10 +-
 .../server/master/service/FailoverService.java     |  33 ++++--
 ...hreadTest.java => WorkflowExecuteTaskTest.java} |  19 ++-
 .../ProcessInstanceExecCacheManagerImplTest.java   |  12 +-
 .../master/registry/MasterRegistryClientTest.java  |   7 --
 .../server/registry/HeartBeatTask.java             |   2 +-
 .../service/registry/RegistryClient.java           |   3 +-
 22 files changed, 273 insertions(+), 362 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index 860e493ee1..0382b53c2c 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -795,7 +795,7 @@ public final class Constants {
      */
     public static final String PSTREE = "pstree";
 
-    public static final Boolean KUBERNETES_MODE = !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_PORT"));
+    public static final boolean KUBERNETES_MODE = !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_HOST")) && !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_PORT"));
 
     /**
      * dry run flag
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 6ab0d4e51a..cb01e14504 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -20,18 +20,8 @@ package org.apache.dolphinscheduler.server.master;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.IStoppable;
 import org.apache.dolphinscheduler.common.thread.Stopper;
-import org.apache.dolphinscheduler.remote.NettyRemotingServer;
-import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
-import org.apache.dolphinscheduler.server.log.LoggerRequestProcessor;
-import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.processor.CacheProcessor;
-import org.apache.dolphinscheduler.server.master.processor.StateEventProcessor;
-import org.apache.dolphinscheduler.server.master.processor.TaskEventProcessor;
-import org.apache.dolphinscheduler.server.master.processor.TaskExecuteResponseProcessor;
-import org.apache.dolphinscheduler.server.master.processor.TaskExecuteRunningProcessor;
-import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor;
 import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
+import org.apache.dolphinscheduler.server.master.rpc.MasterRPCServer;
 import org.apache.dolphinscheduler.server.master.runner.EventExecuteService;
 import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread;
 import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerService;
@@ -58,14 +48,9 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
 public class MasterServer implements IStoppable {
     private static final Logger logger = LoggerFactory.getLogger(MasterServer.class);
 
-    @Autowired
-    private MasterConfig masterConfig;
-
     @Autowired
     private SpringApplicationContext springApplicationContext;
 
-    private NettyRemotingServer nettyRemotingServer;
-
     @Autowired
     private MasterRegistryClient masterRegistryClient;
 
@@ -78,24 +63,6 @@ public class MasterServer implements IStoppable {
     @Autowired
     private Scheduler scheduler;
 
-    @Autowired
-    private TaskExecuteRunningProcessor taskExecuteRunningProcessor;
-
-    @Autowired
-    private TaskExecuteResponseProcessor taskExecuteResponseProcessor;
-
-    @Autowired
-    private TaskEventProcessor taskEventProcessor;
-
-    @Autowired
-    private StateEventProcessor stateEventProcessor;
-
-    @Autowired
-    private CacheProcessor cacheProcessor;
-
-    @Autowired
-    private TaskKillResponseProcessor taskKillResponseProcessor;
-
     @Autowired
     private EventExecuteService eventExecuteService;
 
@@ -103,7 +70,7 @@ public class MasterServer implements IStoppable {
     private FailoverExecuteThread failoverExecuteThread;
 
     @Autowired
-    private LoggerRequestProcessor loggerRequestProcessor;
+    private MasterRPCServer masterRPCServer;
 
     public static void main(String[] args) {
         Thread.currentThread().setName(Constants.THREAD_NAME_MASTER_SERVER);
@@ -115,25 +82,8 @@ public class MasterServer implements IStoppable {
      */
     @PostConstruct
     public void run() throws SchedulerException {
-        // init remoting server
-        NettyServerConfig serverConfig = new NettyServerConfig();
-        serverConfig.setListenPort(masterConfig.getListenPort());
-        this.nettyRemotingServer = new NettyRemotingServer(serverConfig);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE, taskExecuteResponseProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RUNNING, taskExecuteRunningProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_RESPONSE, taskKillResponseProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.STATE_EVENT_REQUEST, stateEventProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_FORCE_STATE_EVENT_REQUEST, taskEventProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_WAKEUP_EVENT_REQUEST, taskEventProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.CACHE_EXPIRE, cacheProcessor);
-
-        // logger server
-        this.nettyRemotingServer.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, loggerRequestProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.ROLL_VIEW_LOG_REQUEST, loggerRequestProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.VIEW_WHOLE_LOG_REQUEST, loggerRequestProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.REMOVE_TAK_LOG_REQUEST, loggerRequestProcessor);
-
-        this.nettyRemotingServer.start();
+        // init rpc server
+        this.masterRPCServer.start();
 
         // install task plugin
         this.taskPluginManager.installPlugin();
@@ -168,6 +118,7 @@ public class MasterServer implements IStoppable {
         try {
             // execute only once
             if (Stopper.isStopped()) {
+                logger.warn("MasterServer has been stopped ..., current cause: {}", cause);
                 return;
             }
 
@@ -184,10 +135,13 @@ public class MasterServer implements IStoppable {
             }
             // close
             this.masterSchedulerService.close();
-            this.nettyRemotingServer.close();
+            this.masterRPCServer.close();
             this.masterRegistryClient.closeRegistry();
-            // close spring Context and will invoke method with @PreDestroy annotation to destory beans. like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc
+            // close spring Context and will invoke method with @PreDestroy annotation to destroy beans.
+            // like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc
             springApplicationContext.close();
+
+            logger.info("MasterServer stopped...");
         } catch (Exception e) {
             logger.error("master server stop exception ", e);
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
index 5fdf64493c..2f5f6dc472 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.cache;
 
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThread;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
 import java.util.Collection;
 
@@ -32,7 +32,7 @@ public interface ProcessInstanceExecCacheManager {
      * @param processInstanceId processInstanceId
      * @return WorkflowExecuteThread
      */
-    WorkflowExecuteThread getByProcessInstanceId(int processInstanceId);
+    WorkflowExecuteRunnable getByProcessInstanceId(int processInstanceId);
 
     /**
      * judge the process instance does it exist
@@ -55,12 +55,12 @@ public interface ProcessInstanceExecCacheManager {
      * @param processInstanceId     processInstanceId
      * @param workflowExecuteThread if it is null, will not be cached
      */
-    void cache(int processInstanceId, WorkflowExecuteThread workflowExecuteThread);
+    void cache(int processInstanceId, WorkflowExecuteRunnable workflowExecuteThread);
 
     /**
      * get all WorkflowExecuteThread from cache
      *
      * @return all WorkflowExecuteThread in cache
      */
-    Collection<WorkflowExecuteThread> getAll();
+    Collection<WorkflowExecuteRunnable> getAll();
 }
\ No newline at end of file
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
index 1d0ab4841a..137bd4ecc5 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
@@ -18,7 +18,7 @@
 package org.apache.dolphinscheduler.server.master.cache.impl;
 
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThread;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
 import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
@@ -33,10 +33,10 @@ import com.google.common.collect.ImmutableList;
 @Component
 public class ProcessInstanceExecCacheManagerImpl implements ProcessInstanceExecCacheManager {
 
-    private final ConcurrentHashMap<Integer, WorkflowExecuteThread> processInstanceExecMaps = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<Integer, WorkflowExecuteRunnable> processInstanceExecMaps = new ConcurrentHashMap<>();
 
     @Override
-    public WorkflowExecuteThread getByProcessInstanceId(int processInstanceId) {
+    public WorkflowExecuteRunnable getByProcessInstanceId(int processInstanceId) {
         return processInstanceExecMaps.get(processInstanceId);
     }
 
@@ -51,7 +51,7 @@ public class ProcessInstanceExecCacheManagerImpl implements ProcessInstanceExecC
     }
 
     @Override
-    public void cache(int processInstanceId, WorkflowExecuteThread workflowExecuteThread) {
+    public void cache(int processInstanceId, WorkflowExecuteRunnable workflowExecuteThread) {
         if (workflowExecuteThread == null) {
             return;
         }
@@ -59,7 +59,7 @@ public class ProcessInstanceExecCacheManagerImpl implements ProcessInstanceExecC
     }
 
     @Override
-    public Collection<WorkflowExecuteThread> getAll() {
+    public Collection<WorkflowExecuteRunnable> getAll() {
         return ImmutableList.copyOf(processInstanceExecMaps.values());
     }
 }
\ No newline at end of file
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
index cba592c63d..83772a1054 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
@@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.StateEventResponseCommand;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThread;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
 
 import java.util.ArrayList;
@@ -135,7 +135,7 @@ public class StateEventResponseService {
                 return;
             }
 
-            WorkflowExecuteThread workflowExecuteThread = this.processInstanceExecCacheManager.getByProcessInstanceId(stateEvent.getProcessInstanceId());
+            WorkflowExecuteRunnable workflowExecuteThread = this.processInstanceExecCacheManager.getByProcessInstanceId(stateEvent.getProcessInstanceId());
             switch (stateEvent.getType()) {
                 case TASK_STATE_CHANGE:
                     workflowExecuteThread.refreshTaskInstance(stateEvent.getTaskInstanceId());
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
similarity index 84%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThread.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
index 47b190e246..593567c6eb 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
@@ -25,11 +25,12 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThread;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
 import org.apache.dolphinscheduler.server.utils.DataQualityResultOperator;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
+import java.util.Optional;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.slf4j.Logger;
@@ -40,9 +41,9 @@ import io.netty.channel.Channel;
 /**
  * task execute thread
  */
-public class TaskExecuteThread {
+public class TaskExecuteRunnable implements Runnable {
 
-    private static final Logger logger = LoggerFactory.getLogger(TaskExecuteThread.class);
+    private static final Logger logger = LoggerFactory.getLogger(TaskExecuteRunnable.class);
 
     private final int processInstanceId;
 
@@ -56,8 +57,8 @@ public class TaskExecuteThread {
 
     private DataQualityResultOperator dataQualityResultOperator;
 
-    public TaskExecuteThread(int processInstanceId, ProcessService processService, WorkflowExecuteThreadPool workflowExecuteThreadPool,
-                             ProcessInstanceExecCacheManager processInstanceExecCacheManager, DataQualityResultOperator dataQualityResultOperator) {
+    public TaskExecuteRunnable(int processInstanceId, ProcessService processService, WorkflowExecuteThreadPool workflowExecuteThreadPool,
+                               ProcessInstanceExecCacheManager processInstanceExecCacheManager, DataQualityResultOperator dataQualityResultOperator) {
         this.processInstanceId = processInstanceId;
         this.processService = processService;
         this.workflowExecuteThreadPool = workflowExecuteThreadPool;
@@ -65,6 +66,7 @@ public class TaskExecuteThread {
         this.dataQualityResultOperator = dataQualityResultOperator;
     }
 
+    @Override
     public void run() {
         while (!this.events.isEmpty()) {
             TaskEvent event = this.events.peek();
@@ -113,12 +115,12 @@ public class TaskExecuteThread {
         int taskInstanceId = taskEvent.getTaskInstanceId();
         int processInstanceId = taskEvent.getProcessInstanceId();
 
-        TaskInstance taskInstance;
-        WorkflowExecuteThread workflowExecuteThread = this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
-        if (workflowExecuteThread != null && workflowExecuteThread.checkTaskInstanceById(taskInstanceId)) {
-            taskInstance = workflowExecuteThread.getTaskInstance(taskInstanceId);
+        Optional<TaskInstance> taskInstance;
+        WorkflowExecuteRunnable workflowExecuteRunnable = this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        if (workflowExecuteRunnable != null && workflowExecuteRunnable.checkTaskInstanceById(taskInstanceId)) {
+            taskInstance = workflowExecuteRunnable.getTaskInstance(taskInstanceId);
         } else {
-            taskInstance = processService.findTaskInstanceById(taskInstanceId);
+            taskInstance = Optional.ofNullable(processService.findTaskInstanceById(taskInstanceId));
         }
 
         switch (event) {
@@ -148,11 +150,12 @@ public class TaskExecuteThread {
     /**
      * handle dispatch event
      */
-    private void handleDispatchEvent(TaskEvent taskEvent, TaskInstance taskInstance) {
-        if (taskInstance == null) {
+    private void handleDispatchEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
+        if (!taskInstanceOptional.isPresent()) {
             logger.error("taskInstance is null");
             return;
         }
+        TaskInstance taskInstance = taskInstanceOptional.get();
         if (taskInstance.getState() != ExecutionStatus.SUBMITTED_SUCCESS) {
             return;
         }
@@ -164,10 +167,11 @@ public class TaskExecuteThread {
     /**
      * handle running event
      */
-    private void handleRunningEvent(TaskEvent taskEvent, TaskInstance taskInstance) {
+    private void handleRunningEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
         Channel channel = taskEvent.getChannel();
         try {
-            if (taskInstance != null) {
+            if (taskInstanceOptional.isPresent()) {
+                TaskInstance taskInstance = taskInstanceOptional.get();
                 if (taskInstance.getState().typeIsFinished()) {
                     logger.warn("task is finish, running event is meaningless, taskInstanceId:{}, state:{}", taskInstance.getId(), taskInstance.getState());
                 } else {
@@ -194,10 +198,11 @@ public class TaskExecuteThread {
     /**
      * handle result event
      */
-    private void handleResultEvent(TaskEvent taskEvent, TaskInstance taskInstance) {
+    private void handleResultEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
         Channel channel = taskEvent.getChannel();
         try {
-            if (taskInstance != null) {
+            if (taskInstanceOptional.isPresent()) {
+                TaskInstance taskInstance = taskInstanceOptional.get();
                 dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance);
 
                 taskInstance.setStartTime(taskEvent.getStartTime());
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
index c9c2868d36..323ea86411 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
@@ -40,7 +40,7 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
 
     private static final Logger logger = LoggerFactory.getLogger(TaskExecuteThreadPool.class);
 
-    private final ConcurrentHashMap<String, TaskExecuteThread> multiThreadFilterMap = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<String, TaskExecuteRunnable> multiThreadFilterMap = new ConcurrentHashMap<>();
 
     @Autowired
     private MasterConfig masterConfig;
@@ -67,7 +67,7 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
     /**
      * task event thread map
      */
-    private final ConcurrentHashMap<Integer, TaskExecuteThread> taskExecuteThreadMap = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<Integer, TaskExecuteRunnable> taskExecuteThreadMap = new ConcurrentHashMap<>();
 
     @PostConstruct
     private void init() {
@@ -83,26 +83,26 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
             return;
         }
         if (!taskExecuteThreadMap.containsKey(taskEvent.getProcessInstanceId())) {
-            TaskExecuteThread taskExecuteThread = new TaskExecuteThread(
+            TaskExecuteRunnable taskExecuteThread = new TaskExecuteRunnable(
                     taskEvent.getProcessInstanceId(),
                     processService, workflowExecuteThreadPool,
                     processInstanceExecCacheManager,
                     dataQualityResultOperator);
             taskExecuteThreadMap.put(taskEvent.getProcessInstanceId(), taskExecuteThread);
         }
-        TaskExecuteThread taskExecuteThread = taskExecuteThreadMap.get(taskEvent.getProcessInstanceId());
-        if (taskExecuteThread != null) {
-            taskExecuteThread.addEvent(taskEvent);
+        TaskExecuteRunnable taskExecuteRunnable= taskExecuteThreadMap.get(taskEvent.getProcessInstanceId());
+        if (taskExecuteRunnable != null) {
+            taskExecuteRunnable.addEvent(taskEvent);
         }
     }
 
     public void eventHandler() {
-        for (TaskExecuteThread taskExecuteThread: taskExecuteThreadMap.values()) {
+        for (TaskExecuteRunnable taskExecuteThread : taskExecuteThreadMap.values()) {
             executeEvent(taskExecuteThread);
         }
     }
 
-    public void executeEvent(TaskExecuteThread taskExecuteThread) {
+    public void executeEvent(TaskExecuteRunnable taskExecuteThread) {
         if (taskExecuteThread.eventSize() == 0) {
             return;
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java
new file mode 100644
index 0000000000..bc1b217f9e
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterConnectionStateListener.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.registry;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.dolphinscheduler.registry.api.ConnectionListener;
+import org.apache.dolphinscheduler.registry.api.ConnectionState;
+import org.apache.dolphinscheduler.service.registry.RegistryClient;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MasterConnectionStateListener implements ConnectionListener {
+
+    private static final Logger logger = LoggerFactory.getLogger(MasterConnectionStateListener.class);
+
+    private final String masterNodePath;
+    private final RegistryClient registryClient;
+
+    public MasterConnectionStateListener(String masterNodePath, RegistryClient registryClient) {
+        this.masterNodePath = checkNotNull(masterNodePath);
+        this.registryClient = checkNotNull(registryClient);
+    }
+
+    @Override
+    public void onUpdate(ConnectionState state) {
+        switch (state) {
+            case CONNECTED:
+                logger.debug("registry connection state is {}", state);
+                break;
+            case SUSPENDED:
+                logger.warn("registry connection state is {}, ready to retry connection", state);
+                break;
+            case RECONNECTED:
+                logger.debug("registry connection state is {}, clean the node info", state);
+                registryClient.remove(masterNodePath);
+                registryClient.persistEphemeral(masterNodePath, "");
+                break;
+            case DISCONNECTED:
+                logger.warn("registry connection state is {}, ready to stop myself", state);
+                registryClient.getStoppable().stop("registry connection state is DISCONNECTED, stop myself");
+                break;
+            default:
+        }
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
index c79f9b7071..5cf1c945af 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
@@ -24,35 +24,18 @@ import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.IStoppable;
 import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.common.model.Server;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.registry.api.ConnectionState;
+import org.apache.dolphinscheduler.registry.api.RegistryException;
 import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
-import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
 import org.apache.dolphinscheduler.server.master.service.FailoverService;
 import org.apache.dolphinscheduler.server.registry.HeartBeatTask;
-import org.apache.dolphinscheduler.server.utils.ProcessUtils;
-import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.registry.RegistryClient;
 
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
-import java.time.Duration;
 import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -65,9 +48,8 @@ import org.springframework.stereotype.Component;
 import com.google.common.collect.Sets;
 
 /**
- * zookeeper master client
- * <p>
- * single instance
+ * <p>DolphinScheduler master register client, used to connect to registry and hand the registry events.
+ * <p>When the Master node startup, it will register in registry center. And schedule a {@link HeartBeatTask} to update its metadata in registry.
  */
 @Component
 public class MasterRegistryClient {
@@ -101,8 +83,10 @@ public class MasterRegistryClient {
      * master startup time, ms
      */
     private long startupTime;
+    private String masterAddress;
 
     public void init() {
+        this.masterAddress = NetUtils.getAddr(masterConfig.getListenPort());
         this.startupTime = System.currentTimeMillis();
         this.heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HeartBeatExecutor"));
     }
@@ -111,11 +95,12 @@ public class MasterRegistryClient {
         try {
             // master registry
             registry();
-
+            registryClient.addConnectionStateListener(new MasterConnectionStateListener(getCurrentNodePath(),
+                                                                                        registryClient));
             registryClient.subscribe(REGISTRY_DOLPHINSCHEDULER_NODE, new MasterRegistryDataListener());
         } catch (Exception e) {
             logger.error("master start up exception", e);
-            throw new RuntimeException("master start up error", e);
+            throw new RegistryException("Master registry client start up error", e);
         }
     }
 
@@ -198,24 +183,25 @@ public class MasterRegistryClient {
     }
 
     /**
-     * registry
+     * Registry the current master server itself to registry.
      */
-    public void registry() {
-        String address = NetUtils.getAddr(masterConfig.getListenPort());
-        String localNodePath = getMasterPath();
+    void registry() {
+        logger.info("master node : {} registering to registry center...", masterAddress);
+        String localNodePath = getCurrentNodePath();
         int masterHeartbeatInterval = masterConfig.getHeartbeatInterval();
         HeartBeatTask heartBeatTask = new HeartBeatTask(startupTime,
-                masterConfig.getMaxCpuLoadAvg(),
-                masterConfig.getReservedMemory(),
-                Sets.newHashSet(getMasterPath()),
-                Constants.MASTER_TYPE,
-                registryClient);
+                                                        masterConfig.getMaxCpuLoadAvg(),
+                                                        masterConfig.getReservedMemory(),
+                                                        Sets.newHashSet(localNodePath),
+                                                        Constants.MASTER_TYPE,
+                                                        registryClient);
 
         // remove before persist
         registryClient.remove(localNodePath);
         registryClient.persistEphemeral(localNodePath, heartBeatTask.getHeartBeatInfo());
 
         while (!registryClient.checkNodeExists(NetUtils.getHost(), NodeType.MASTER)) {
+            logger.warn("The current master server node:{} cannot find in registry....", NetUtils.getHost());
             ThreadUtils.sleep(SLEEP_TIME_MILLIS);
         }
 
@@ -225,37 +211,17 @@ public class MasterRegistryClient {
         // delete dead server
         registryClient.handleDeadServer(Collections.singleton(localNodePath), NodeType.MASTER, Constants.DELETE_OP);
 
-        registryClient.addConnectionStateListener(this::handleConnectionState);
-        this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, masterHeartbeatInterval, masterHeartbeatInterval, TimeUnit.SECONDS);
-        logger.info("master node : {} registry to ZK successfully with heartBeatInterval : {}s", address, masterHeartbeatInterval);
-
-    }
+        this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, 0L, masterHeartbeatInterval, TimeUnit.SECONDS);
+        logger.info("master node : {} registry to ZK successfully with heartBeatInterval : {}s",
+                    masterAddress,
+                    masterHeartbeatInterval);
 
-    public void handleConnectionState(ConnectionState state) {
-        String localNodePath = getMasterPath();
-        switch (state) {
-            case CONNECTED:
-                logger.debug("registry connection state is {}", state);
-                break;
-            case SUSPENDED:
-                logger.warn("registry connection state is {}, ready to retry connection", state);
-                break;
-            case RECONNECTED:
-                logger.debug("registry connection state is {}, clean the node info", state);
-                registryClient.persistEphemeral(localNodePath, "");
-                break;
-            case DISCONNECTED:
-                logger.warn("registry connection state is {}, ready to stop myself", state);
-                registryClient.getStoppable().stop("registry connection state is DISCONNECTED, stop myself");
-                break;
-            default:
-        }
     }
 
     public void deregister() {
         try {
             String address = getLocalAddress();
-            String localNodePath = getMasterPath();
+            String localNodePath = getCurrentNodePath();
             registryClient.remove(localNodePath);
             logger.info("master node : {} unRegistry to register center.", address);
             heartBeatExecutor.shutdown();
@@ -269,7 +235,7 @@ public class MasterRegistryClient {
     /**
      * get master path
      */
-    private String getMasterPath() {
+    private String getCurrentNodePath() {
         String address = getLocalAddress();
         return REGISTRY_DOLPHINSCHEDULER_MASTERS + "/" + address;
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java
similarity index 51%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java
index 6ab0d4e51a..e0b65dd077 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java
@@ -15,11 +15,8 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.server.master;
+package org.apache.dolphinscheduler.server.master.rpc;
 
-import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.IStoppable;
-import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.remote.NettyRemotingServer;
 import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
@@ -31,52 +28,27 @@ import org.apache.dolphinscheduler.server.master.processor.TaskEventProcessor;
 import org.apache.dolphinscheduler.server.master.processor.TaskExecuteResponseProcessor;
 import org.apache.dolphinscheduler.server.master.processor.TaskExecuteRunningProcessor;
 import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor;
-import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
-import org.apache.dolphinscheduler.server.master.runner.EventExecuteService;
-import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread;
-import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerService;
-import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
-import org.apache.dolphinscheduler.service.task.TaskPluginManager;
+import org.apache.dolphinscheduler.server.master.processor.TaskRecallProcessor;
 
 import javax.annotation.PostConstruct;
 
-import org.quartz.Scheduler;
-import org.quartz.SchedulerException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.cache.annotation.EnableCaching;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.transaction.annotation.EnableTransactionManagement;
-
-@SpringBootApplication
-@ComponentScan("org.apache.dolphinscheduler")
-@EnableTransactionManagement
-@EnableCaching
-public class MasterServer implements IStoppable {
-    private static final Logger logger = LoggerFactory.getLogger(MasterServer.class);
+import org.springframework.stereotype.Service;
 
-    @Autowired
-    private MasterConfig masterConfig;
+/**
+ * Master RPC Server, used to send/receive request to other system.
+ */
+@Service
+public class MasterRPCServer implements AutoCloseable {
 
-    @Autowired
-    private SpringApplicationContext springApplicationContext;
+    private static final Logger logger = LoggerFactory.getLogger(MasterRPCServer.class);
 
     private NettyRemotingServer nettyRemotingServer;
 
     @Autowired
-    private MasterRegistryClient masterRegistryClient;
-
-    @Autowired
-    private TaskPluginManager taskPluginManager;
-
-    @Autowired
-    private MasterSchedulerService masterSchedulerService;
-
-    @Autowired
-    private Scheduler scheduler;
+    private MasterConfig masterConfig;
 
     @Autowired
     private TaskExecuteRunningProcessor taskExecuteRunningProcessor;
@@ -97,24 +69,13 @@ public class MasterServer implements IStoppable {
     private TaskKillResponseProcessor taskKillResponseProcessor;
 
     @Autowired
-    private EventExecuteService eventExecuteService;
-
-    @Autowired
-    private FailoverExecuteThread failoverExecuteThread;
+    private TaskRecallProcessor taskRecallProcessor;
 
     @Autowired
     private LoggerRequestProcessor loggerRequestProcessor;
 
-    public static void main(String[] args) {
-        Thread.currentThread().setName(Constants.THREAD_NAME_MASTER_SERVER);
-        SpringApplication.run(MasterServer.class);
-    }
-
-    /**
-     * run master server
-     */
     @PostConstruct
-    public void run() throws SchedulerException {
+    private void init() {
         // init remoting server
         NettyServerConfig serverConfig = new NettyServerConfig();
         serverConfig.setListenPort(masterConfig.getListenPort());
@@ -126,6 +87,7 @@ public class MasterServer implements IStoppable {
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_FORCE_STATE_EVENT_REQUEST, taskEventProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_WAKEUP_EVENT_REQUEST, taskEventProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.CACHE_EXPIRE, cacheProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_RECALL, taskRecallProcessor);
 
         // logger server
         this.nettyRemotingServer.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, loggerRequestProcessor);
@@ -134,67 +96,19 @@ public class MasterServer implements IStoppable {
         this.nettyRemotingServer.registerProcessor(CommandType.REMOVE_TAK_LOG_REQUEST, loggerRequestProcessor);
 
         this.nettyRemotingServer.start();
-
-        // install task plugin
-        this.taskPluginManager.installPlugin();
-
-        // self tolerant
-        this.masterRegistryClient.init();
-        this.masterRegistryClient.start();
-        this.masterRegistryClient.setRegistryStoppable(this);
-
-        this.masterSchedulerService.init();
-        this.masterSchedulerService.start();
-
-        this.eventExecuteService.start();
-        this.failoverExecuteThread.start();
-
-        this.scheduler.start();
-
-        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
-            if (Stopper.isRunning()) {
-                close("shutdownHook");
-            }
-        }));
     }
 
-    /**
-     * gracefully close
-     *
-     * @param cause close cause
-     */
-    public void close(String cause) {
-
-        try {
-            // execute only once
-            if (Stopper.isStopped()) {
-                return;
-            }
-
-            logger.info("master server is stopping ..., cause : {}", cause);
-
-            // set stop signal is true
-            Stopper.stop();
-
-            try {
-                // thread sleep 3 seconds for thread quietly stop
-                Thread.sleep(3000L);
-            } catch (Exception e) {
-                logger.warn("thread sleep exception ", e);
-            }
-            // close
-            this.masterSchedulerService.close();
-            this.nettyRemotingServer.close();
-            this.masterRegistryClient.closeRegistry();
-            // close spring Context and will invoke method with @PreDestroy annotation to destory beans. like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc
-            springApplicationContext.close();
-        } catch (Exception e) {
-            logger.error("master server stop exception ", e);
-        }
+    public void start() {
+        logger.info("Starting Master RPC Server...");
+        this.nettyRemotingServer.start();
+        logger.info("Started Master RPC Server...");
     }
 
     @Override
-    public void stop(String cause) {
-        close(cause);
+    public void close() {
+        logger.info("Closing Master RPC Server...");
+        this.nettyRemotingServer.close();
+        logger.info("Closed Master RPC Server...");
     }
+
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
index 6f6f718c2f..fc9c4fd854 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
@@ -62,7 +62,7 @@ public class EventExecuteService extends Thread {
     }
 
     private void eventHandler() {
-        for (WorkflowExecuteThread workflowExecuteThread : this.processInstanceExecCacheManager.getAll()) {
+        for (WorkflowExecuteRunnable workflowExecuteThread : this.processInstanceExecCacheManager.getAll()) {
             workflowExecuteThreadPool.executeEvent(workflowExecuteThread);
         }
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
index aa894f8b3b..5fd812f162 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
@@ -56,6 +56,8 @@ public class FailoverExecuteThread extends Thread {
         logger.info("failover execute thread started");
         while (Stopper.isRunning()) {
             try {
+                // todo: DO we need to schedule a task to do this kind of check
+                // This kind of check may only need to be executed when a master server start
                 failoverService.checkMasterFailover();
             } catch (Exception e) {
                 logger.error("failover execute error", e);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index 5c97c0b41f..e688863d06 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -165,7 +165,7 @@ public class MasterSchedulerService extends Thread {
                 continue;
             }
 
-            WorkflowExecuteThread workflowExecuteThread = new WorkflowExecuteThread(
+            WorkflowExecuteRunnable workflowExecuteThread = new WorkflowExecuteRunnable(
                     processInstance
                     , processService
                     , nettyExecutorManager
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index ba614e573b..7635209ebb 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
 
 import org.apache.hadoop.util.ThreadUtil;
 
+import java.util.Optional;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.slf4j.Logger;
@@ -112,7 +113,7 @@ public class StateWheelExecuteThread extends Thread {
             if (processInstanceId == null) {
                 continue;
             }
-            WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
             if (workflowExecuteThread == null) {
                 logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}", processInstanceId);
                 processInstanceTimeoutCheckList.remove(processInstanceId);
@@ -219,20 +220,21 @@ public class StateWheelExecuteThread extends Thread {
             int processInstanceId = taskInstanceKey.getProcessInstanceId();
             long taskCode = taskInstanceKey.getTaskCode();
 
-            WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
             if (workflowExecuteThread == null) {
                 logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
                         processInstanceId, taskCode);
                 taskInstanceTimeoutCheckList.remove(taskInstanceKey);
                 continue;
             }
-            TaskInstance taskInstance = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
-            if (taskInstance == null) {
+            Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
+            if (!taskInstanceOptional.isPresent()) {
                 logger.warn("can not find taskInstance from workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
                         processInstanceId, taskCode);
                 taskInstanceTimeoutCheckList.remove(taskInstanceKey);
                 continue;
             }
+            TaskInstance taskInstance = taskInstanceOptional.get();
             if (TimeoutFlag.OPEN == taskInstance.getTaskDefine().getTimeoutFlag()) {
                 long timeRemain = DateUtils.getRemainTime(taskInstance.getStartTime(), (long) taskInstance.getTaskDefine().getTimeout() * Constants.SEC_2_MINUTES_TIME_UNIT);
                 if (timeRemain < 0) {
@@ -252,7 +254,7 @@ public class StateWheelExecuteThread extends Thread {
             int processInstanceId = taskInstanceKey.getProcessInstanceId();
             long taskCode = taskInstanceKey.getTaskCode();
 
-            WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
 
             if (workflowExecuteThread == null) {
                 logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
@@ -261,7 +263,7 @@ public class StateWheelExecuteThread extends Thread {
                 continue;
             }
 
-            TaskInstance taskInstance = workflowExecuteThread.getRetryTaskInstanceByTaskCode(taskCode);
+            Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getRetryTaskInstanceByTaskCode(taskCode);
             ProcessInstance processInstance = workflowExecuteThread.getProcessInstance();
 
             if (processInstance.getState() == ExecutionStatus.READY_STOP) {
@@ -270,13 +272,14 @@ public class StateWheelExecuteThread extends Thread {
                 break;
             }
 
-            if (taskInstance == null) {
+            if (!taskInstanceOptional.isPresent()) {
                 logger.warn("can not find taskInstance from workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
                         processInstanceId, taskCode);
                 taskInstanceRetryCheckList.remove(taskInstanceKey);
                 continue;
             }
 
+            TaskInstance taskInstance = taskInstanceOptional.get();
             if (taskInstance.retryTaskIntervalOverTime()) {
                 // reset taskInstance endTime and state
                 // todo relative funtion: TaskInstance.retryTaskIntervalOverTime, WorkflowExecuteThread.cloneRetryTaskInstance
@@ -297,20 +300,21 @@ public class StateWheelExecuteThread extends Thread {
             int processInstanceId = taskInstanceKey.getProcessInstanceId();
             long taskCode = taskInstanceKey.getTaskCode();
 
-            WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
             if (workflowExecuteThread == null) {
                 logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
                         processInstanceId, taskCode);
                 taskInstanceStateCheckList.remove(taskInstanceKey);
                 continue;
             }
-            TaskInstance taskInstance = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
-            if (taskInstance == null) {
+            Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
+            if (!taskInstanceOptional.isPresent()) {
                 logger.warn("can not find taskInstance from workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
                         processInstanceId, taskCode);
                 taskInstanceStateCheckList.remove(taskInstanceKey);
                 continue;
             }
+            TaskInstance taskInstance = taskInstanceOptional.get();
             if (taskInstance.getState().typeIsFinished()) {
                 continue;
             }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
similarity index 96%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index fede5abad2..67bfe3c928 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -100,34 +100,34 @@ import org.slf4j.LoggerFactory;
 import com.google.common.collect.Lists;
 
 /**
- * master exec thread,split dag
+ * Workflow execute task, used to execute a workflow instance.
  */
-public class WorkflowExecuteThread {
+public class WorkflowExecuteRunnable implements Runnable {
 
     /**
      * logger of WorkflowExecuteThread
      */
-    private static final Logger logger = LoggerFactory.getLogger(WorkflowExecuteThread.class);
+    private static final Logger logger = LoggerFactory.getLogger(WorkflowExecuteRunnable.class);
 
     /**
      * master config
      */
-    private MasterConfig masterConfig;
+    private final MasterConfig masterConfig;
 
     /**
      * process service
      */
-    private ProcessService processService;
+    private final ProcessService processService;
 
     /**
      * alert manager
      */
-    private ProcessAlertManager processAlertManager;
+    private final ProcessAlertManager processAlertManager;
 
     /**
      * netty executor manager
      */
-    private NettyExecutorManager nettyExecutorManager;
+    private final NettyExecutorManager nettyExecutorManager;
 
     /**
      * process instance
@@ -162,7 +162,7 @@ public class WorkflowExecuteThread {
     /**
      * task instance hash map, taskId as key
      */
-    private Map<Integer, TaskInstance> taskInstanceMap = new ConcurrentHashMap<>();
+    private final Map<Integer, TaskInstance> taskInstanceMap = new ConcurrentHashMap<>();
 
     /**
      * running taskProcessor, taskCode as key, taskProcessor as value
@@ -174,34 +174,34 @@ public class WorkflowExecuteThread {
      * valid task map, taskCode as key, taskId as value
      * in a DAG, only one taskInstance per taskCode is valid
      */
-    private Map<Long, Integer> validTaskMap = new ConcurrentHashMap<>();
+    private final Map<Long, Integer> validTaskMap = new ConcurrentHashMap<>();
 
     /**
      * error task map, taskCode as key, taskInstanceId as value
      * in a DAG, only one taskInstance per taskCode is valid
      */
-    private Map<Long, Integer> errorTaskMap = new ConcurrentHashMap<>();
+    private final Map<Long, Integer> errorTaskMap = new ConcurrentHashMap<>();
 
     /**
      * complete task map, taskCode as key, taskInstanceId as value
      * in a DAG, only one taskInstance per taskCode is valid
      */
-    private Map<Long, Integer> completeTaskMap = new ConcurrentHashMap<>();
+    private final Map<Long, Integer> completeTaskMap = new ConcurrentHashMap<>();
 
     /**
      * depend failed task map, taskCode as key, taskId as value
      */
-    private Map<Long, Integer> dependFailedTaskMap = new ConcurrentHashMap<>();
+    private final Map<Long, Integer> dependFailedTaskMap = new ConcurrentHashMap<>();
 
     /**
      * forbidden task map, code as key
      */
-    private Map<Long, TaskNode> forbiddenTaskMap = new ConcurrentHashMap<>();
+    private final Map<Long, TaskNode> forbiddenTaskMap = new ConcurrentHashMap<>();
 
     /**
      * skip task map, code as key
      */
-    private Map<String, TaskNode> skipTaskNodeMap = new ConcurrentHashMap<>();
+    private final Map<String, TaskNode> skipTaskNodeMap = new ConcurrentHashMap<>();
 
     /**
      * complement date list
@@ -211,27 +211,25 @@ public class WorkflowExecuteThread {
     /**
      * state event queue
      */
-    private ConcurrentLinkedQueue<StateEvent> stateEvents = new ConcurrentLinkedQueue<>();
+    private final ConcurrentLinkedQueue<StateEvent> stateEvents = new ConcurrentLinkedQueue<>();
 
     /**
      * ready to submit task queue
      */
-    private PeerTaskInstancePriorityQueue readyToSubmitTaskQueue = new PeerTaskInstancePriorityQueue();
+    private final PeerTaskInstancePriorityQueue readyToSubmitTaskQueue = new PeerTaskInstancePriorityQueue();
 
     /**
      * wait to retry taskInstance map, taskCode as key, taskInstance as value
      * before retry, the taskInstance id is 0
      */
-    private Map<Long, TaskInstance> waitToRetryTaskInstanceMap = new ConcurrentHashMap<>();
+    private final Map<Long, TaskInstance> waitToRetryTaskInstanceMap = new ConcurrentHashMap<>();
 
     /**
      * state wheel execute thread
      */
-    private StateWheelExecuteThread stateWheelExecuteThread;
+    private final StateWheelExecuteThread stateWheelExecuteThread;
 
     /**
-     * constructor of WorkflowExecuteThread
-     *
      * @param processInstance         processInstance
      * @param processService          processService
      * @param nettyExecutorManager    nettyExecutorManager
@@ -239,12 +237,12 @@ public class WorkflowExecuteThread {
      * @param masterConfig            masterConfig
      * @param stateWheelExecuteThread stateWheelExecuteThread
      */
-    public WorkflowExecuteThread(ProcessInstance processInstance,
-                                 ProcessService processService,
-                                 NettyExecutorManager nettyExecutorManager,
-                                 ProcessAlertManager processAlertManager,
-                                 MasterConfig masterConfig,
-                                 StateWheelExecuteThread stateWheelExecuteThread) {
+    public WorkflowExecuteRunnable(ProcessInstance processInstance
+            , ProcessService processService
+            , NettyExecutorManager nettyExecutorManager
+            , ProcessAlertManager processAlertManager
+            , MasterConfig masterConfig
+            , StateWheelExecuteThread stateWheelExecuteThread) {
         this.processService = processService;
         this.processInstance = processInstance;
         this.masterConfig = masterConfig;
@@ -401,7 +399,10 @@ public class WorkflowExecuteThread {
             return true;
         }
 
-        TaskInstance task = getTaskInstance(stateEvent.getTaskInstanceId());
+        Optional<TaskInstance> taskInstanceOptional = getTaskInstance(stateEvent.getTaskInstanceId());
+        TaskInstance task = taskInstanceOptional.orElseThrow(
+                () -> new RuntimeException("Cannot find task instance by task instance id: " + stateEvent.getTaskInstanceId()));
+
         if (task.getState() == null) {
             logger.error("task state is null, state handler error: {}", stateEvent);
             return true;
@@ -632,37 +633,37 @@ public class WorkflowExecuteThread {
     /**
      * get task instance from memory
      */
-    public TaskInstance getTaskInstance(int taskInstanceId) {
+    public Optional<TaskInstance> getTaskInstance(int taskInstanceId) {
         if (taskInstanceMap.containsKey(taskInstanceId)) {
-            return taskInstanceMap.get(taskInstanceId);
+            return Optional.ofNullable(taskInstanceMap.get(taskInstanceId));
         }
-        return null;
+        return Optional.empty();
     }
 
-    public TaskInstance getTaskInstance(long taskCode) {
-        if (taskInstanceMap == null || taskInstanceMap.size() == 0) {
-            return null;
+    public Optional<TaskInstance> getTaskInstance(long taskCode) {
+        if (taskInstanceMap.isEmpty()) {
+            return Optional.empty();
         }
         for (TaskInstance taskInstance : taskInstanceMap.values()) {
             if (taskInstance.getTaskCode() == taskCode) {
-                return taskInstance;
+                return Optional.of(taskInstance);
             }
         }
-        return null;
+        return Optional.empty();
     }
 
-    public TaskInstance getActiveTaskInstanceByTaskCode(long taskCode) {
+    public Optional<TaskInstance> getActiveTaskInstanceByTaskCode(long taskCode) {
         if (activeTaskProcessorMaps.containsKey(taskCode)) {
-            return activeTaskProcessorMaps.get(taskCode).taskInstance();
+            return Optional.ofNullable(activeTaskProcessorMaps.get(taskCode).taskInstance());
         }
-        return null;
+        return Optional.empty();
     }
 
-    public TaskInstance getRetryTaskInstanceByTaskCode(long taskCode) {
+    public Optional<TaskInstance> getRetryTaskInstanceByTaskCode(long taskCode) {
         if (waitToRetryTaskInstanceMap.containsKey(taskCode)) {
-            return waitToRetryTaskInstanceMap.get(taskCode);
+            return Optional.ofNullable(waitToRetryTaskInstanceMap.get(taskCode));
         }
-        return null;
+        return Optional.empty();
     }
 
     private boolean processStateChangeHandler(StateEvent stateEvent) {
@@ -697,7 +698,9 @@ public class WorkflowExecuteThread {
 
     private boolean processBlockHandler(StateEvent stateEvent) {
         try {
-            TaskInstance task = getTaskInstance(stateEvent.getTaskInstanceId());
+            Optional<TaskInstance> taskInstanceOptional = getTaskInstance(stateEvent.getTaskInstanceId());
+            TaskInstance task = taskInstanceOptional.orElseThrow(
+                    () -> new RuntimeException("Cannot find taskInstance by taskInstanceId:" + stateEvent.getTaskInstanceId()));
             if (!checkTaskInstanceByStateEvent(stateEvent)) {
                 logger.error("task {} is not a blocking task", task.getTaskCode());
                 return false;
@@ -790,9 +793,10 @@ public class WorkflowExecuteThread {
     }
 
     /**
-     * process start handle
+     * ProcessInstance start entrypoint.
      */
-    public void startProcess() {
+    @Override
+    public void run() {
         if (this.taskInstanceMap.size() > 0) {
             return;
         }
@@ -1305,9 +1309,9 @@ public class WorkflowExecuteThread {
         List<TaskInstance> taskInstances = new ArrayList<>();
         for (String taskNode : submitTaskNodeList) {
             TaskNode taskNodeObject = dag.getNode(taskNode);
-            TaskInstance existTaskInstance = getTaskInstance(taskNodeObject.getCode());
-            if (existTaskInstance != null) {
-                taskInstances.add(existTaskInstance);
+            Optional<TaskInstance> existTaskInstanceOptional = getTaskInstance(taskNodeObject.getCode());
+            if (existTaskInstanceOptional.isPresent()) {
+                taskInstances.add(existTaskInstanceOptional.get());
                 continue;
             }
             TaskInstance task = createTaskInstance(processInstance, taskNodeObject);
@@ -2024,8 +2028,6 @@ public class WorkflowExecuteThread {
         }
     }
 
-}
-
     private void measureTaskState(StateEvent taskStateEvent) {
         if (taskStateEvent == null || taskStateEvent.getExecutionStatus() == null) {
             // the event is broken
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index 556f36a1b7..c3d771e365 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -68,7 +68,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     /**
      * multi-thread filter, avoid handling workflow at the same time
      */
-    private ConcurrentHashMap<String, WorkflowExecuteThread> multiThreadFilterMap = new ConcurrentHashMap();
+    private ConcurrentHashMap<String, WorkflowExecuteRunnable> multiThreadFilterMap = new ConcurrentHashMap();
 
     @PostConstruct
     private void init() {
@@ -82,7 +82,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
      * submit state event
      */
     public void submitStateEvent(StateEvent stateEvent) {
-        WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(stateEvent.getProcessInstanceId());
+        WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(stateEvent.getProcessInstanceId());
         if (workflowExecuteThread == null) {
             logger.warn("workflowExecuteThread is null, stateEvent:{}", stateEvent);
             return;
@@ -93,14 +93,14 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     /**
      * start workflow
      */
-    public void startWorkflow(WorkflowExecuteThread workflowExecuteThread) {
-        submit(workflowExecuteThread::startProcess);
+    public void startWorkflow(WorkflowExecuteRunnable workflowExecuteThread) {
+        submit(workflowExecuteThread);
     }
 
     /**
      * execute workflow
      */
-    public void executeEvent(final WorkflowExecuteRunnable workflowExecuteThread) {
+    public void executeEvent(WorkflowExecuteRunnable workflowExecuteThread) {
         if (!workflowExecuteThread.isStart() || workflowExecuteThread.eventSize() == 0) {
             return;
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
index 2557b659e0..dad2dbfe1e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
@@ -17,6 +17,9 @@
 
 package org.apache.dolphinscheduler.server.master.service;
 
+import io.micrometer.core.annotation.Counted;
+import io.micrometer.core.annotation.Timed;
+
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.NodeType;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
@@ -36,13 +39,15 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.registry.RegistryClient;
 
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.StopWatch;
 
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -70,12 +75,14 @@ public class FailoverService {
     /**
      * check master failover
      */
+    @Counted(value = "failover_scheduler_check_task_count")
+    @Timed(value = "failover_scheduler_check_task_time", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
     public void checkMasterFailover() {
         List<String> hosts = getNeedFailoverMasterServers();
         if (CollectionUtils.isEmpty(hosts)) {
             return;
         }
-        LOGGER.info("need failover hosts:{}", hosts);
+        LOGGER.info("{} begin to failover hosts:{}", getLocalAddress(), hosts);
 
         for (String host : hosts) {
             failoverMasterWithLock(host);
@@ -114,9 +121,9 @@ public class FailoverService {
     }
 
     /**
-     * failover master
-     * <p>
-     * failover process instance and associated task instance
+     * Failover master, will failover process instance and associated task instance.
+     * <p>When the process instance belongs to the given masterHost and the restartTime is before the current server start up time,
+     * then the process instance will be failovered.
      *
      * @param masterHost master host
      */
@@ -125,11 +132,11 @@ public class FailoverService {
             return;
         }
         Date serverStartupTime = getServerStartupTime(NodeType.MASTER, masterHost);
-        long startTime = System.currentTimeMillis();
+        StopWatch failoverTimeCost = StopWatch.createStarted();
         List<ProcessInstance> needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(masterHost);
-        LOGGER.info("start master[{}] failover, process list size:{}", masterHost, needFailoverProcessInstanceList.size());
+        LOGGER.info("start master[{}] failover, need to failover process list size:{}", masterHost, needFailoverProcessInstanceList.size());
 
-        // servers need to contains master hosts and worker hosts, otherwise the logic task will failover fail.
+        // servers need to contain master hosts and worker hosts, otherwise the logic task will failover fail.
         List<Server> servers = registryClient.getServerList(NodeType.WORKER);
         servers.addAll(registryClient.getServerList(NodeType.MASTER));
 
@@ -145,7 +152,7 @@ public class FailoverService {
             }
 
             if (serverStartupTime != null && processInstance.getRestartTime() != null
-                && processInstance.getRestartTime().after(serverStartupTime)) {
+                    && processInstance.getRestartTime().after(serverStartupTime)) {
                 continue;
             }
 
@@ -155,7 +162,8 @@ public class FailoverService {
             processService.processNeedFailoverProcessInstances(processInstance);
         }
 
-        LOGGER.info("master[{}] failover end, useTime:{}ms", masterHost, System.currentTimeMillis() - startTime);
+        failoverTimeCost.stop();
+        LOGGER.info("master[{}] failover end, useTime:{}ms", masterHost, failoverTimeCost.getTime(TimeUnit.MILLISECONDS));
     }
 
     /**
@@ -248,7 +256,10 @@ public class FailoverService {
     }
 
     /**
-     * get need failover master servers
+     * Get need failover master servers.
+     * <p>
+     * Query the process instances from database, if the processInstance's host doesn't exist in registry
+     * or the host is the currentServer, then it will need to failover.
      *
      * @return need failover master servers
      */
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
similarity index 95%
rename from dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
index 6e1c84382d..339c5e80ae 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteThreadTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
@@ -21,7 +21,6 @@ import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_D
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
-
 import static org.powermock.api.mockito.PowerMockito.mock;
 
 import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -37,7 +36,7 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.runner.StateWheelExecuteThread;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThread;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -70,10 +69,10 @@ import org.springframework.context.ApplicationContext;
  * test for WorkflowExecuteThread
  */
 @RunWith(PowerMockRunner.class)
-@PrepareForTest({WorkflowExecuteThread.class})
-public class WorkflowExecuteThreadTest {
+@PrepareForTest({WorkflowExecuteRunnable.class})
+public class WorkflowExecuteTaskTest {
 
-    private WorkflowExecuteThread workflowExecuteThread;
+    private WorkflowExecuteRunnable workflowExecuteThread;
 
     private ProcessInstance processInstance;
 
@@ -118,9 +117,9 @@ public class WorkflowExecuteThreadTest {
         Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition);
 
         stateWheelExecuteThread = mock(StateWheelExecuteThread.class);
-        workflowExecuteThread = PowerMockito.spy(new WorkflowExecuteThread(processInstance, processService, null, null, config, stateWheelExecuteThread));
+        workflowExecuteThread = PowerMockito.spy(new WorkflowExecuteRunnable(processInstance, processService, null, null, config, stateWheelExecuteThread));
         // prepareProcess init dag
-        Field dag = WorkflowExecuteThread.class.getDeclaredField("dag");
+        Field dag = WorkflowExecuteRunnable.class.getDeclaredField("dag");
         dag.setAccessible(true);
         dag.set(workflowExecuteThread, new DAG());
         PowerMockito.doNothing().when(workflowExecuteThread, "endProcess");
@@ -132,7 +131,7 @@ public class WorkflowExecuteThreadTest {
             Map<String, String> cmdParam = new HashMap<>();
             cmdParam.put(CMD_PARAM_START_NODES, "1,2,3");
             Mockito.when(processInstance.getCommandParam()).thenReturn(JSONUtils.toJsonString(cmdParam));
-            Class<WorkflowExecuteThread> masterExecThreadClass = WorkflowExecuteThread.class;
+            Class<WorkflowExecuteRunnable> masterExecThreadClass = WorkflowExecuteRunnable.class;
             Method method = masterExecThreadClass.getDeclaredMethod("parseStartNodeName", String.class);
             method.setAccessible(true);
             List<String> nodeNames = (List<String>) method.invoke(workflowExecuteThread, JSONUtils.toJsonString(cmdParam));
@@ -158,7 +157,7 @@ public class WorkflowExecuteThreadTest {
             Mockito.when(processService.findTaskInstanceByIdList(
                     Arrays.asList(taskInstance1.getId(), taskInstance2.getId(), taskInstance3.getId(), taskInstance4.getId()))
             ).thenReturn(Arrays.asList(taskInstance1, taskInstance2, taskInstance3, taskInstance4));
-            Class<WorkflowExecuteThread> masterExecThreadClass = WorkflowExecuteThread.class;
+            Class<WorkflowExecuteRunnable> masterExecThreadClass = WorkflowExecuteRunnable.class;
             Method method = masterExecThreadClass.getDeclaredMethod("getStartTaskInstanceList", String.class);
             method.setAccessible(true);
             List<TaskInstance> taskInstances = (List<TaskInstance>) method.invoke(workflowExecuteThread, JSONUtils.toJsonString(cmdParam));
@@ -202,7 +201,7 @@ public class WorkflowExecuteThreadTest {
             completeTaskList.put(taskInstance1.getTaskCode(), taskInstance1.getId());
             completeTaskList.put(taskInstance2.getTaskCode(), taskInstance2.getId());
 
-            Class<WorkflowExecuteThread> masterExecThreadClass = WorkflowExecuteThread.class;
+            Class<WorkflowExecuteRunnable> masterExecThreadClass = WorkflowExecuteRunnable.class;
 
             Field completeTaskMapField = masterExecThreadClass.getDeclaredField("completeTaskMap");
             completeTaskMapField.setAccessible(true);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
index 6fa9116e63..d7b53f3c34 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.cache.impl;
 
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThread;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
 import java.util.Collection;
 
@@ -37,7 +37,7 @@ public class ProcessInstanceExecCacheManagerImplTest {
     private ProcessInstanceExecCacheManagerImpl processInstanceExecCacheManager;
 
     @Mock
-    private WorkflowExecuteThread workflowExecuteThread;
+    private WorkflowExecuteRunnable workflowExecuteThread;
 
     @Before
     public void before() {
@@ -47,7 +47,7 @@ public class ProcessInstanceExecCacheManagerImplTest {
 
     @Test
     public void testGetByProcessInstanceId() {
-        WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(1);
+        WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(1);
         Assert.assertEquals("workflowExecuteThread1", workflowExecuteThread.getKey());
     }
 
@@ -59,20 +59,20 @@ public class ProcessInstanceExecCacheManagerImplTest {
     @Test
     public void testCacheNull() {
         processInstanceExecCacheManager.cache(2, null);
-        WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(2);
+        WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(2);
         Assert.assertNull(workflowExecuteThread);
     }
 
     @Test
     public void testRemoveByProcessInstanceId() {
         processInstanceExecCacheManager.removeByProcessInstanceId(1);
-        WorkflowExecuteThread workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(1);
+        WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(1);
         Assert.assertNull(workflowExecuteThread);
     }
 
     @Test
     public void testGetAll() {
-        Collection<WorkflowExecuteThread> workflowExecuteThreads = processInstanceExecCacheManager.getAll();
+        Collection<WorkflowExecuteRunnable> workflowExecuteThreads = processInstanceExecCacheManager.getAll();
         Assert.assertEquals(1, workflowExecuteThreads.size());
     }
 }
\ No newline at end of file
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
index d3d5b03de9..2a4b4c33ff 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
@@ -111,13 +111,6 @@ public class MasterRegistryClientTest {
         masterRegistryClient.registry();
     }
 
-    @Test
-    public void handleConnectionStateTest() {
-        masterRegistryClient.handleConnectionState(ConnectionState.CONNECTED);
-        masterRegistryClient.handleConnectionState(ConnectionState.RECONNECTED);
-        masterRegistryClient.handleConnectionState(ConnectionState.SUSPENDED);
-    }
-
     @Test
     public void removeNodePathTest() {
         masterRegistryClient.removeMasterNodePath("/path", NodeType.MASTER, false);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
index 17b4df14a9..129aaff0de 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
@@ -89,7 +89,7 @@ public class HeartBeatTask implements Runnable {
                 registryClient.persistEphemeral(heartBeatPath, heartBeat.encodeHeartBeat());
             }
         } catch (Throwable ex) {
-            logger.error("error write heartbeat info", ex);
+            logger.error("HeartBeat task execute failed", ex);
         }
     }
 }
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
index 2f98b09b34..c606007e88 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.service.registry;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static org.apache.dolphinscheduler.common.Constants.ADD_OP;
 import static org.apache.dolphinscheduler.common.Constants.COLON;
 import static org.apache.dolphinscheduler.common.Constants.DELETE_OP;
@@ -29,8 +30,6 @@ import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH;
 import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
 import static org.apache.dolphinscheduler.common.Constants.WORKER_TYPE;
 
-import static com.google.common.base.Preconditions.checkArgument;
-
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.IStoppable;
 import org.apache.dolphinscheduler.common.enums.NodeType;


[dolphinscheduler] 19/29: Fix compile error

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit da03ef32e93daf103d44845db21f5614ebd9cfe7
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jul 19 12:00:37 2022 +0800

    Fix compile error
---
 .../apache/dolphinscheduler/service/process/ProcessServiceTest.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
index e19a992b83..a98138dd03 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
@@ -371,7 +371,7 @@ public class ProcessServiceTest {
         command5.setCommandType(CommandType.START_PROCESS);
         command5.setDryRun(Constants.DRY_RUN_FLAG_NO);
         Mockito.when(commandMapper.deleteById(5)).thenReturn(1);
-        ProcessInstance processInstance1 = processService.handleCommand(logger, host, command5);
+        ProcessInstance processInstance1 = processService.handleCommand(host, command5);
         Assert.assertTrue(processInstance1.getGlobalParams().contains("\"testStartParam1\""));
 
         ProcessDefinition processDefinition1 = new ProcessDefinition();
@@ -426,7 +426,7 @@ public class ProcessServiceTest {
         command7.setProcessDefinitionVersion(1);
         Mockito.when(commandMapper.deleteById(7)).thenReturn(1);
         Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(11L, 1, Constants.RUNNING_PROCESS_STATE, 224)).thenReturn(null);
-        ProcessInstance processInstance8 = processService.handleCommand(logger, host, command7);
+        ProcessInstance processInstance8 = processService.handleCommand(host, command7);
         Assert.assertTrue(processInstance8 == null);
 
         ProcessDefinition processDefinition2 = new ProcessDefinition();


[dolphinscheduler] 17/29: [Fix-10785] Fix state event handle error will not retry (#10786)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 7500e9968296f62269daca95b270fc65870fce97
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Wed Jul 6 14:53:28 2022 +0800

    [Fix-10785] Fix state event handle error will not retry (#10786)
    
    * Fix state event handle error will not retry
    
    * Use state event handler to deal with the event
    
    (cherry picked from commit 67d14fb7b3d941af613a96a0b9c3a2928e5c201c)
---
 .../api/service/impl/ExecutorServiceImpl.java      |  12 +-
 .../master/consumer/TaskPriorityQueueConsumer.java |   2 +
 .../master/dispatch/host/assign/Selector.java      |   2 +-
 .../server/master/event}/StateEvent.java           |   3 +-
 .../StateEventHandleError.java}                    |  23 +-
 .../StateEventHandleException.java}                |  22 +-
 .../server/master/event/StateEventHandler.java     |  38 +-
 .../master/event/StateEventHandlerManager.java     |  41 +-
 .../master/event/TaskRetryStateEventHandler.java   |  47 +++
 .../server/master/event/TaskStateEventHandler.java | 115 ++++++
 .../master/event/TaskTimeoutStateEventHandler.java |  64 +++
 .../event/TaskWaitTaskGroupStateHandler.java       |  42 +-
 .../event/WorkflowBlockStateEventHandler.java      |  60 +++
 .../master/event/WorkflowStateEventHandler.java    |  95 +++++
 .../event/WorkflowTimeoutStateEventHandler.java    |  39 ++
 .../master/processor/StateEventProcessor.java      |   5 +-
 .../master/processor/TaskEventProcessor.java       |   2 +-
 .../processor/queue/StateEventResponseService.java |   3 +-
 .../master/processor/queue/TaskEventService.java   |  17 +-
 .../processor/queue/TaskExecuteRunnable.java       |   3 +-
 .../processor/queue/TaskExecuteThreadPool.java     |  21 +-
 .../master/runner/MasterSchedulerService.java      |   1 +
 .../master/runner/StateWheelExecuteThread.java     |   2 +-
 .../master/runner/WorkflowExecuteRunnable.java     | 441 ++++++---------------
 .../master/runner/WorkflowExecuteThreadPool.java   |   3 +-
 .../server/master/service/FailoverService.java     |   2 +-
 .../server/master/service/FailoverServiceTest.java |   2 +-
 .../remote/processor/NettyRemoteChannel.java       |   7 +-
 .../processor/StateEventCallbackService.java       |  29 +-
 .../queue/PeerTaskInstancePriorityQueue.java       |   3 +-
 .../service/queue/TaskPriorityQueue.java           |   2 +-
 .../service/queue/TaskPriorityQueueImpl.java       |   3 +-
 .../processor/TaskExecuteResponseAckProcessor.java |   8 +-
 33 files changed, 683 insertions(+), 476 deletions(-)

diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
index 5f72be37ed..b58cc9dbcd 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
@@ -65,6 +65,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
 import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
+import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
 
@@ -116,7 +117,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
     private ProcessService processService;
 
     @Autowired
-    StateEventCallbackService stateEventCallbackService;
+    private StateEventCallbackService stateEventCallbackService;
 
     @Autowired
     private TaskDefinitionMapper taskDefinitionMapper;
@@ -467,13 +468,12 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
 
         // determine whether the process is normal
         if (update > 0) {
-            String host = processInstance.getHost();
-            String address = host.split(":")[0];
-            int port = Integer.parseInt(host.split(":")[1]);
+            // directly send the process instance state change event to target master, not guarantee the event send success
             StateEventChangeCommand stateEventChangeCommand = new StateEventChangeCommand(
                     processInstance.getId(), 0, processInstance.getState(), processInstance.getId(), 0
             );
-            stateEventCallbackService.sendResult(address, port, stateEventChangeCommand.convert2Command());
+            Host host = new Host(processInstance.getHost());
+            stateEventCallbackService.sendResult(host, stateEventChangeCommand.convert2Command());
             putMsg(result, Status.SUCCESS);
         } else {
             putMsg(result, Status.EXECUTE_PROCESS_INSTANCE_ERROR);
@@ -737,7 +737,7 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
                         }
                     }
                     logger.info("In parallel mode, current expectedParallelismNumber:{}", createCount);
-                    
+
                     // Distribute the number of tasks equally to each command.
                     // The last command with insufficient quantity will be assigned to the remaining tasks.
                     int itemsPerCommand = (listDateSize / createCount);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index e8cf4b2919..f04ae15dd8 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -116,7 +116,9 @@ public class TaskPriorityQueueConsumer extends BaseDaemonThread {
     @PostConstruct
     public void init() {
         this.consumerThreadPoolExecutor = (ThreadPoolExecutor) ThreadUtils.newDaemonFixedThreadExecutor("TaskUpdateQueueConsumerThread", masterConfig.getDispatchTaskNumber());
+        logger.info("Task priority queue consume thread staring");
         super.start();
+        logger.info("Task priority queue consume thread started");
     }
 
     @Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
index 8eed9d991e..49a4053e5e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
@@ -27,7 +27,7 @@ public interface Selector<T> {
 
     /**
      * select
-     * @param source source
+     * @param source source, the given source should not be empty.
      * @return T
      */
     T select(Collection<T> source);
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java
similarity index 91%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java
index 7f4be924dd..68cd582994 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEvent.java
@@ -15,8 +15,9 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
+import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 
 import io.netty.channel.Channel;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleError.java
similarity index 63%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleError.java
index 8eed9d991e..a8dce04f03 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleError.java
@@ -15,20 +15,19 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.server.master.dispatch.host.assign;
-
-import java.util.Collection;
+package org.apache.dolphinscheduler.server.master.event;
 
 /**
- * selector
- * @param <T> T
+ * This exception represent the exception can not recover, this happens when the event is broken.
+ * And when we get this exception, we will drop the event.
  */
-public interface Selector<T> {
+public class StateEventHandleError extends Exception {
+
+    public StateEventHandleError(String message) {
+        super(message);
+    }
 
-    /**
-     * select
-     * @param source source
-     * @return T
-     */
-    T select(Collection<T> source);
+    public StateEventHandleError(String message, Throwable throwable) {
+        super(message, throwable);
+    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleException.java
similarity index 65%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleException.java
index 8eed9d991e..dc9456a397 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/Selector.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleException.java
@@ -15,20 +15,18 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.server.master.dispatch.host.assign;
-
-import java.util.Collection;
+package org.apache.dolphinscheduler.server.master.event;
 
 /**
- * selector
- * @param <T> T
+ * This exception represent the exception can be recovered, when we get this exception, we will retry the event.
  */
-public interface Selector<T> {
+public class StateEventHandleException extends Exception {
+
+    public StateEventHandleException(String message) {
+        super(message);
+    }
 
-    /**
-     * select
-     * @param source source
-     * @return T
-     */
-    T select(Collection<T> source);
+    public StateEventHandleException(String message, Throwable throwable) {
+        super(message, throwable);
+    }
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
similarity index 51%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
index 7f4be924dd..00808b2e29 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
@@ -15,36 +15,22 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
-import io.netty.channel.Channel;
-import lombok.Data;
-
-/**
- * state event
- */
-@Data
-public class StateEvent {
+public interface StateEventHandler {
 
     /**
-     * origin_pid-origin_task_id-process_instance_id-task_instance_id
+     * Handle a event, if handle success will reture true, else return false
+     *
+     * @param stateEvent given state event.
+     * @throws StateEventHandleException this exception means it can be recovered.
+     * @throws StateEventHandleError     this exception means it cannot be recovered, so the event need to drop.
      */
-    private String key;
-
-    private StateEventType type;
-
-    private ExecutionStatus executionStatus;
-
-    private int taskInstanceId;
-
-    private long taskCode;
-
-    private int processInstanceId;
-
-    private String context;
-
-    private Channel channel;
+    boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent)
+        throws StateEventHandleException, StateEventHandleError;
 
+    StateEventType getEventType();
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java
similarity index 51%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java
index 7f4be924dd..b436b55890 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java
@@ -15,36 +15,27 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.enums.StateEventType;
 
-import io.netty.channel.Channel;
-import lombok.Data;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.ServiceLoader;
 
-/**
- * state event
- */
-@Data
-public class StateEvent {
-
-    /**
-     * origin_pid-origin_task_id-process_instance_id-task_instance_id
-     */
-    private String key;
-
-    private StateEventType type;
-
-    private ExecutionStatus executionStatus;
-
-    private int taskInstanceId;
-
-    private long taskCode;
+public class StateEventHandlerManager {
 
-    private int processInstanceId;
+    private static final Map<StateEventType, StateEventHandler> stateEventHandlerMap = new HashMap<>();
 
-    private String context;
+    static {
+        ServiceLoader.load(StateEventHandler.class)
+            .forEach(stateEventHandler -> stateEventHandlerMap.put(stateEventHandler.getEventType(),
+                stateEventHandler));
+    }
 
-    private Channel channel;
+    public static Optional<StateEventHandler> getStateEventHandler(StateEventType stateEventType) {
+        return Optional.ofNullable(stateEventHandlerMap.get(stateEventType));
+    }
 
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java
new file mode 100644
index 0000000000..ee8168856a
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRetryStateEventHandler.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+
+import java.util.Map;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(StateEventHandler.class)
+public class TaskRetryStateEventHandler implements StateEventHandler {
+    @Override
+    public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent)
+        throws StateEventHandleException {
+        TaskMetrics.incTaskRetry();
+        Map<Long, TaskInstance> waitToRetryTaskInstanceMap = workflowExecuteRunnable.getWaitToRetryTaskInstanceMap();
+        TaskInstance taskInstance = waitToRetryTaskInstanceMap.get(stateEvent.getTaskCode());
+        workflowExecuteRunnable.addTaskToStandByList(taskInstance);
+        workflowExecuteRunnable.submitStandByTask();
+        waitToRetryTaskInstanceMap.remove(stateEvent.getTaskCode());
+        return true;
+    }
+
+    @Override
+    public StateEventType getEventType() {
+        return StateEventType.TASK_RETRY;
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java
new file mode 100644
index 0000000000..e3ad268f97
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessor;
+import org.apache.dolphinscheduler.server.master.runner.task.TaskAction;
+
+import java.util.Map;
+import java.util.Optional;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(StateEventHandler.class)
+public class TaskStateEventHandler implements StateEventHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(TaskStateEventHandler.class);
+
+    @Override
+    public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent)
+        throws StateEventHandleException, StateEventHandleError {
+        measureTaskState(stateEvent);
+        workflowExecuteRunnable.checkTaskInstanceByStateEvent(stateEvent);
+
+        Optional<TaskInstance> taskInstanceOptional =
+            workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId());
+
+        TaskInstance task = taskInstanceOptional.orElseThrow(() -> new StateEventHandleError(
+            "Cannot find task instance from taskMap by task instance id: " + stateEvent.getTaskInstanceId()));
+
+        if (task.getState() == null) {
+            throw new StateEventHandleError("Task state event handle error due to task state is null");
+        }
+
+        Map<Long, Integer> completeTaskMap = workflowExecuteRunnable.getCompleteTaskMap();
+
+        if (task.getState().typeIsFinished()) {
+            if (completeTaskMap.containsKey(task.getTaskCode())
+                && completeTaskMap.get(task.getTaskCode()) == task.getId()) {
+                logger.warn("The task instance is already complete, stateEvent: {}", stateEvent);
+                return true;
+            }
+            workflowExecuteRunnable.taskFinished(task);
+            if (task.getTaskGroupId() > 0) {
+                workflowExecuteRunnable.releaseTaskGroup(task);
+            }
+            return true;
+        }
+        Map<Long, ITaskProcessor> activeTaskProcessMap = workflowExecuteRunnable.getActiveTaskProcessMap();
+        if (activeTaskProcessMap.containsKey(task.getTaskCode())) {
+            ITaskProcessor iTaskProcessor = activeTaskProcessMap.get(task.getTaskCode());
+            iTaskProcessor.action(TaskAction.RUN);
+
+            if (iTaskProcessor.taskInstance().getState().typeIsFinished()) {
+                if (iTaskProcessor.taskInstance().getState() != task.getState()) {
+                    task.setState(iTaskProcessor.taskInstance().getState());
+                }
+                workflowExecuteRunnable.taskFinished(task);
+            }
+            return true;
+        }
+        throw new StateEventHandleException(
+            "Task state event handle error, due to the task is not in activeTaskProcessorMaps");
+    }
+
+    @Override
+    public StateEventType getEventType() {
+        return StateEventType.TASK_STATE_CHANGE;
+    }
+
+    private void measureTaskState(StateEvent taskStateEvent) {
+        if (taskStateEvent == null || taskStateEvent.getExecutionStatus() == null) {
+            // the event is broken
+            logger.warn("The task event is broken..., taskEvent: {}", taskStateEvent);
+            return;
+        }
+        if (taskStateEvent.getExecutionStatus().typeIsFinished()) {
+            TaskMetrics.incTaskFinish();
+        }
+        switch (taskStateEvent.getExecutionStatus()) {
+            case STOP:
+                TaskMetrics.incTaskStop();
+                break;
+            case SUCCESS:
+                TaskMetrics.incTaskSuccess();
+                break;
+            case FAILURE:
+                TaskMetrics.incTaskFailure();
+                break;
+            default:
+                break;
+        }
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java
new file mode 100644
index 0000000000..240f10ff2c
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskTimeoutStateEventHandler.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessor;
+import org.apache.dolphinscheduler.server.master.runner.task.TaskAction;
+
+import java.util.Map;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(StateEventHandler.class)
+public class TaskTimeoutStateEventHandler implements StateEventHandler {
+    @Override
+    public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent)
+        throws StateEventHandleError {
+        TaskMetrics.incTaskTimeout();
+        workflowExecuteRunnable.checkTaskInstanceByStateEvent(stateEvent);
+
+        TaskInstance taskInstance = workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId()).get();
+
+        if (TimeoutFlag.CLOSE == taskInstance.getTaskDefine().getTimeoutFlag()) {
+            return true;
+        }
+        TaskTimeoutStrategy taskTimeoutStrategy = taskInstance.getTaskDefine().getTimeoutNotifyStrategy();
+        Map<Long, ITaskProcessor> activeTaskProcessMap = workflowExecuteRunnable.getActiveTaskProcessMap();
+        if (TaskTimeoutStrategy.FAILED == taskTimeoutStrategy
+            || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) {
+            ITaskProcessor taskProcessor = activeTaskProcessMap.get(taskInstance.getTaskCode());
+            taskProcessor.action(TaskAction.TIMEOUT);
+        }
+        if (TaskTimeoutStrategy.WARN == taskTimeoutStrategy || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) {
+            workflowExecuteRunnable.processTimeout();
+            workflowExecuteRunnable.taskTimeout(taskInstance);
+        }
+        return true;
+    }
+
+    @Override
+    public StateEventType getEventType() {
+        return StateEventType.TASK_TIMEOUT;
+    }
+}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
similarity index 54%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
index 7f4be924dd..9a3c59a949 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
@@ -15,36 +15,22 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
-import io.netty.channel.Channel;
-import lombok.Data;
+import com.google.auto.service.AutoService;
 
-/**
- * state event
- */
-@Data
-public class StateEvent {
-
-    /**
-     * origin_pid-origin_task_id-process_instance_id-task_instance_id
-     */
-    private String key;
-
-    private StateEventType type;
-
-    private ExecutionStatus executionStatus;
-
-    private int taskInstanceId;
-
-    private long taskCode;
-
-    private int processInstanceId;
-
-    private String context;
-
-    private Channel channel;
+@AutoService(StateEventHandler.class)
+public class TaskWaitTaskGroupStateHandler implements StateEventHandler {
+    @Override
+    public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent) {
+        return workflowExecuteRunnable.checkForceStartAndWakeUp(stateEvent);
+    }
 
+    @Override
+    public StateEventType getEventType() {
+        return StateEventType.WAIT_TASK_GROUP;
+    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java
new file mode 100644
index 0000000000..f7349fcbd1
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowBlockStateEventHandler.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+
+import java.util.Optional;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(StateEventHandler.class)
+public class WorkflowBlockStateEventHandler implements StateEventHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(WorkflowBlockStateEventHandler.class);
+
+    @Override
+    public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent)
+        throws StateEventHandleError {
+        Optional<TaskInstance> taskInstanceOptional =
+            workflowExecuteRunnable.getTaskInstance(stateEvent.getTaskInstanceId());
+        if (!taskInstanceOptional.isPresent()) {
+            throw new StateEventHandleError("Cannot find taskInstance from taskMap by taskInstanceId: "
+                + stateEvent.getTaskInstanceId());
+        }
+        TaskInstance task = taskInstanceOptional.get();
+
+        BlockingParameters parameters = JSONUtils.parseObject(task.getTaskParams(), BlockingParameters.class);
+        if (parameters != null && parameters.isAlertWhenBlocking()) {
+            workflowExecuteRunnable.processBlock();
+        }
+        return true;
+    }
+
+    @Override
+    public StateEventType getEventType() {
+        return StateEventType.PROCESS_BLOCKED;
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java
new file mode 100644
index 0000000000..37d8ceb1da
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(StateEventHandler.class)
+public class WorkflowStateEventHandler implements StateEventHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(WorkflowStateEventHandler.class);
+
+    @Override
+    public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent)
+        throws StateEventHandleException {
+        measureProcessState(stateEvent);
+        ProcessInstance processInstance = workflowExecuteRunnable.getProcessInstance();
+        ProcessDefinition processDefinition = processInstance.getProcessDefinition();
+
+        logger.info("process:{} state {} change to {}",
+            processInstance.getId(),
+            processInstance.getState(),
+            stateEvent.getExecutionStatus());
+
+        if (stateEvent.getExecutionStatus() == ExecutionStatus.STOP) {
+            // serial wait execution type needs to wake up the waiting process
+            if (processDefinition.getExecutionType().typeIsSerialWait() || processDefinition.getExecutionType()
+                .typeIsSerialPriority()) {
+                workflowExecuteRunnable.endProcess();
+                return true;
+            }
+            workflowExecuteRunnable.updateProcessInstanceState(stateEvent);
+            return true;
+        }
+        if (workflowExecuteRunnable.processComplementData()) {
+            return true;
+        }
+        if (stateEvent.getExecutionStatus().typeIsFinished()) {
+            workflowExecuteRunnable.endProcess();
+        }
+        if (processInstance.getState() == ExecutionStatus.READY_STOP) {
+            workflowExecuteRunnable.killAllTasks();
+        }
+
+        return true;
+    }
+
+    @Override
+    public StateEventType getEventType() {
+        return StateEventType.PROCESS_STATE_CHANGE;
+    }
+
+    private void measureProcessState(StateEvent processStateEvent) {
+        if (processStateEvent.getExecutionStatus().typeIsFinished()) {
+            ProcessInstanceMetrics.incProcessInstanceFinish();
+        }
+        switch (processStateEvent.getExecutionStatus()) {
+            case STOP:
+                ProcessInstanceMetrics.incProcessInstanceStop();
+                break;
+            case SUCCESS:
+                ProcessInstanceMetrics.incProcessInstanceSuccess();
+                break;
+            case FAILURE:
+                ProcessInstanceMetrics.incProcessInstanceFailure();
+                break;
+            default:
+                break;
+        }
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java
new file mode 100644
index 0000000000..c2fc873bdc
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowTimeoutStateEventHandler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(StateEventHandler.class)
+public class WorkflowTimeoutStateEventHandler implements StateEventHandler {
+    @Override
+    public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent) {
+        ProcessInstanceMetrics.incProcessInstanceTimeout();
+        workflowExecuteRunnable.processTimeout();
+        return true;
+    }
+
+    @Override
+    public StateEventType getEventType() {
+        return StateEventType.PROCESS_TIMEOUT;
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
index b2c47e4112..f90277d0eb 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.processor;
 
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
@@ -62,7 +62,8 @@ public class StateEventProcessor implements NettyRequestProcessor {
         }
         stateEvent.setProcessInstanceId(stateEventChangeCommand.getDestProcessInstanceId());
         stateEvent.setTaskInstanceId(stateEventChangeCommand.getDestTaskInstanceId());
-        StateEventType type = stateEvent.getTaskInstanceId() == 0 ? StateEventType.PROCESS_STATE_CHANGE : StateEventType.TASK_STATE_CHANGE;
+        StateEventType
+            type = stateEvent.getTaskInstanceId() == 0 ? StateEventType.PROCESS_STATE_CHANGE : StateEventType.TASK_STATE_CHANGE;
         stateEvent.setType(type);
 
         try {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
index e597f1cae5..42d4ab4db7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.processor;
 
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
index 4ca6b9eccb..bdf80bbee9 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
@@ -98,6 +98,7 @@ public class StateEventResponseService {
      */
     public void addResponse(StateEvent stateEvent) {
         try {
+            // check the event is validated
             eventQueue.put(stateEvent);
         } catch (InterruptedException e) {
             logger.error("Put state event : {} error", stateEvent, e);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java
index bed3b3d9ed..d4b97c09c7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java
@@ -63,11 +63,15 @@ public class TaskEventService {
 
     @PostConstruct
     public void start() {
-        this.taskEventThread = new TaskEventThread();
+        this.taskEventThread = new TaskEventDispatchThread();
+        logger.info("TaskEvent dispatch thread starting");
         this.taskEventThread.start();
+        logger.info("TaskEvent dispatch thread started");
 
         this.taskEventHandlerThread = new TaskEventHandlerThread();
+        logger.info("TaskEvent handle thread staring");
         this.taskEventHandlerThread.start();
+        logger.info("TaskEvent handle thread started");
     }
 
     @PreDestroy
@@ -94,14 +98,14 @@ public class TaskEventService {
      * @param taskEvent taskEvent
      */
     public void addEvent(TaskEvent taskEvent) {
-        taskExecuteThreadPool.submitTaskEvent(taskEvent);
+        eventQueue.add(taskEvent);
     }
 
     /**
-     * task worker thread
+     * Dispatch event to target task runnable.
      */
-    class TaskEventThread extends BaseDaemonThread {
-        protected TaskEventThread() {
+    class TaskEventDispatchThread extends BaseDaemonThread {
+        protected TaskEventDispatchThread() {
             super("TaskEventLoopThread");
         }
 
@@ -109,7 +113,7 @@ public class TaskEventService {
         public void run() {
             while (Stopper.isRunning()) {
                 try {
-                    // if not task , blocking here
+                    // if not task event, blocking here
                     TaskEvent taskEvent = eventQueue.take();
                     taskExecuteThreadPool.submitTaskEvent(taskEvent);
                 } catch (InterruptedException e) {
@@ -141,6 +145,7 @@ public class TaskEventService {
                     TimeUnit.MILLISECONDS.sleep(Constants.SLEEP_TIME_MILLIS);
                 } catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
+                    logger.warn("TaskEvent handle thread interrupted, will return this loop");
                     break;
                 } catch (Exception e) {
                     logger.error("event handler thread error", e);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
index 3bf249ed3e..f488fc3de1 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
@@ -18,7 +18,7 @@
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
 import org.apache.dolphinscheduler.common.enums.Event;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@@ -189,6 +189,7 @@ public class TaskExecuteRunnable implements Runnable {
                 }
             }
             // if taskInstance is null (maybe deleted) or finish. retry will be meaningless . so ack success
+            // send ack to worker
             TaskExecuteRunningAckCommand taskExecuteRunningAckCommand = new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
             channel.writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
         } catch (Exception e) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
index 68e4511c42..ae578d4c03 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
@@ -82,18 +82,13 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
             logger.warn("Cannot find workflowExecuteThread from cacheManager, event: {}", taskEvent);
             return;
         }
-        if (!taskExecuteThreadMap.containsKey(taskEvent.getProcessInstanceId())) {
-            TaskExecuteRunnable taskExecuteThread = new TaskExecuteRunnable(
-                    taskEvent.getProcessInstanceId(),
-                    processService, workflowExecuteThreadPool,
-                    processInstanceExecCacheManager,
-                    dataQualityResultOperator);
-            taskExecuteThreadMap.put(taskEvent.getProcessInstanceId(), taskExecuteThread);
-        }
-        TaskExecuteRunnable taskExecuteRunnable= taskExecuteThreadMap.get(taskEvent.getProcessInstanceId());
-        if (taskExecuteRunnable != null) {
-            taskExecuteRunnable.addEvent(taskEvent);
-        }
+        TaskExecuteRunnable taskExecuteRunnable = taskExecuteThreadMap.computeIfAbsent(taskEvent.getProcessInstanceId(),
+            (processInstanceId) -> new TaskExecuteRunnable(processInstanceId,
+                processService,
+                workflowExecuteThreadPool,
+                processInstanceExecCacheManager,
+                dataQualityResultOperator));
+        taskExecuteRunnable.addEvent(taskEvent);
     }
 
     public void eventHandler() {
@@ -103,7 +98,7 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
     }
 
     public void executeEvent(TaskExecuteRunnable taskExecuteThread) {
-        if (taskExecuteThread.eventSize() == 0) {
+        if (taskExecuteThread.isEmpty()) {
             return;
         }
         if (multiThreadFilterMap.containsKey(taskExecuteThread.getKey())) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index ae7b4bdeb0..35d3155f43 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -35,6 +35,7 @@ import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
 import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
 import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
+import org.apache.dolphinscheduler.service.expand.CuringParamsService;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
 import org.apache.commons.collections4.CollectionUtils;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index a56a7d8c5a..c70fb1f1d4 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -18,7 +18,7 @@
 package org.apache.dolphinscheduler.server.master.runner;
 
 import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 88d7391dde..7a15a43d2b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -35,15 +35,14 @@ import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.FailureStrategy;
 import org.apache.dolphinscheduler.common.enums.Flag;
 import org.apache.dolphinscheduler.common.enums.Priority;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.enums.TaskDependType;
 import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
-import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
 import org.apache.dolphinscheduler.common.graph.DAG;
 import org.apache.dolphinscheduler.common.model.TaskNode;
 import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
 import org.apache.dolphinscheduler.common.process.ProcessDag;
+import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
@@ -63,14 +62,16 @@ import org.apache.dolphinscheduler.dao.utils.DagHelper;
 import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
 import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
 import org.apache.dolphinscheduler.plugin.task.api.model.Property;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.BlockingParameters;
 import org.apache.dolphinscheduler.remote.command.HostUpdateCommand;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
-import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEventHandleError;
+import org.apache.dolphinscheduler.server.master.event.StateEventHandleException;
+import org.apache.dolphinscheduler.server.master.event.StateEventHandler;
+import org.apache.dolphinscheduler.server.master.event.StateEventHandlerManager;
 import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessor;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskAction;
@@ -275,16 +276,37 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         if (!isStart) {
             return;
         }
+        StateEvent stateEvent = null;
         while (!this.stateEvents.isEmpty()) {
             try {
-                StateEvent stateEvent = this.stateEvents.peek();
-                LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
-                if (stateEventHandler(stateEvent)) {
+                stateEvent = this.stateEvents.peek();
+                LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(),
+                                                            stateEvent.getTaskInstanceId());
+                // if state handle success then will remove this state, otherwise will retry this state next time.
+                // The state should always handle success except database error.
+                checkProcessInstance(stateEvent);
+
+                StateEventHandler stateEventHandler
+                    = StateEventHandlerManager.getStateEventHandler(stateEvent.getType())
+                    .orElseThrow(() -> new StateEventHandleError("Cannot find handler for the given state event"));
+                if (stateEventHandler.handleStateEvent(this, stateEvent)) {
                     this.stateEvents.remove(stateEvent);
                 }
+            } catch (StateEventHandleError stateEventHandleError) {
+                logger.error("State event handle error, will remove this event: {}", stateEvent, stateEventHandleError);
+                this.stateEvents.remove(stateEvent);
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
+            } catch (StateEventHandleException stateEventHandleException) {
+                logger.error("State event handle error, will retry this event: {}",
+                             stateEvent,
+                             stateEventHandleException);
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
             } catch (Exception e) {
                 // we catch the exception here, since if the state event handle failed, the state event will still keep in the stateEvents queue.
-                logger.error("state handle error:", e);
+                logger.error("State event handle error, get a unknown exception, will retry this event: {}",
+                             stateEvent,
+                             e);
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
             } finally {
                 LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
             }
@@ -321,58 +343,14 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         return this.processInstance;
     }
 
-    private boolean stateEventHandler(StateEvent stateEvent) {
-        logger.info("process event: {}", stateEvent);
-
-        if (!checkProcessInstance(stateEvent)) {
-            return false;
-        }
-
-        boolean result = false;
-        switch (stateEvent.getType()) {
-            case PROCESS_STATE_CHANGE:
-                measureProcessState(stateEvent);
-                result = processStateChangeHandler(stateEvent);
-                break;
-            case TASK_STATE_CHANGE:
-                measureTaskState(stateEvent);
-                result = taskStateChangeHandler(stateEvent);
-                break;
-            case PROCESS_TIMEOUT:
-                ProcessInstanceMetrics.incProcessInstanceTimeout();
-                result = processTimeout();
-                break;
-            case TASK_TIMEOUT:
-                TaskMetrics.incTaskTimeout();
-                result = taskTimeout(stateEvent);
-                break;
-            case WAIT_TASK_GROUP:
-                result = checkForceStartAndWakeUp(stateEvent);
-                break;
-            case TASK_RETRY:
-                TaskMetrics.incTaskRetry();
-                result = taskRetryEventHandler(stateEvent);
-                break;
-            case PROCESS_BLOCKED:
-                result = processBlockHandler(stateEvent);
-                break;
-            default:
-                break;
-        }
-
-        if (result) {
-            this.stateEvents.remove(stateEvent);
-        }
-        return result;
-    }
-
-    private boolean checkForceStartAndWakeUp(StateEvent stateEvent) {
+    public boolean checkForceStartAndWakeUp(StateEvent stateEvent) {
         TaskGroupQueue taskGroupQueue = this.processService.loadTaskGroupQueue(stateEvent.getTaskInstanceId());
         if (taskGroupQueue.getForceStart() == Flag.YES.getCode()) {
             TaskInstance taskInstance = this.processService.findTaskInstanceById(stateEvent.getTaskInstanceId());
             ITaskProcessor taskProcessor = activeTaskProcessorMaps.get(taskInstance.getTaskCode());
             taskProcessor.action(TaskAction.DISPATCH);
-            this.processService.updateTaskGroupQueueStatus(taskGroupQueue.getTaskId(), TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode());
+            this.processService.updateTaskGroupQueueStatus(taskGroupQueue.getTaskId(),
+                                                           TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode());
             return true;
         }
         if (taskGroupQueue.getInQueue() == Flag.YES.getCode()) {
@@ -387,79 +365,20 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         return false;
     }
 
-    private boolean taskTimeout(StateEvent stateEvent) {
-        if (!checkTaskInstanceByStateEvent(stateEvent)) {
-            return true;
-        }
-
-        TaskInstance taskInstance = taskInstanceMap.get(stateEvent.getTaskInstanceId());
-        if (TimeoutFlag.CLOSE == taskInstance.getTaskDefine().getTimeoutFlag()) {
-            return true;
-        }
-        TaskTimeoutStrategy taskTimeoutStrategy = taskInstance.getTaskDefine().getTimeoutNotifyStrategy();
-        if (TaskTimeoutStrategy.FAILED == taskTimeoutStrategy || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) {
-            ITaskProcessor taskProcessor = activeTaskProcessorMaps.get(taskInstance.getTaskCode());
-            taskProcessor.action(TaskAction.TIMEOUT);
-        }
-        if (TaskTimeoutStrategy.WARN == taskTimeoutStrategy || TaskTimeoutStrategy.WARNFAILED == taskTimeoutStrategy) {
-            ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(processInstance.getId());
-            processAlertManager.sendTaskTimeoutAlert(processInstance, taskInstance, projectUser);
-        }
-        return true;
-    }
-
-    private boolean processTimeout() {
+    public void processTimeout() {
         ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(processInstance.getId());
         this.processAlertManager.sendProcessTimeoutAlert(this.processInstance, projectUser);
-        return true;
     }
 
-    private boolean taskStateChangeHandler(StateEvent stateEvent) {
-        if (!checkTaskInstanceByStateEvent(stateEvent)) {
-            return true;
-        }
-
-        Optional<TaskInstance> taskInstanceOptional = getTaskInstance(stateEvent.getTaskInstanceId());
-        TaskInstance task = taskInstanceOptional.orElseThrow(
-                () -> new RuntimeException("Cannot find task instance by task instance id: " + stateEvent.getTaskInstanceId()));
-
-        if (task.getState() == null) {
-            logger.error("task state is null, state handler error: {}", stateEvent);
-            return true;
-        }
-
-        if (task.getState().typeIsFinished()) {
-            if (completeTaskMap.containsKey(task.getTaskCode()) && completeTaskMap.get(task.getTaskCode()) == task.getId()) {
-                logger.warn("The task instance is already complete, stateEvent: {}", stateEvent);
-                return true;
-            }
-            taskFinished(task);
-            if (task.getTaskGroupId() > 0) {
-                releaseTaskGroup(task);
-            }
-            return true;
-        }
-        if (activeTaskProcessorMaps.containsKey(task.getTaskCode())) {
-            ITaskProcessor iTaskProcessor = activeTaskProcessorMaps.get(task.getTaskCode());
-            iTaskProcessor.action(TaskAction.RUN);
-
-            if (iTaskProcessor.taskInstance().getState().typeIsFinished()) {
-                if (iTaskProcessor.taskInstance().getState() != task.getState()) {
-                    task.setState(iTaskProcessor.taskInstance().getState());
-                }
-                taskFinished(task);
-            }
-            return true;
-        }
-        logger.error("state handler error: {}", stateEvent);
-
-        return true;
+    public void taskTimeout(TaskInstance taskInstance) {
+        ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(processInstance.getId());
+        processAlertManager.sendTaskTimeoutAlert(processInstance, taskInstance, projectUser);
     }
 
-    private void taskFinished(TaskInstance taskInstance) {
+    public void taskFinished(TaskInstance taskInstance) throws StateEventHandleException {
         logger.info("TaskInstance finished task code:{} state:{} ",
-            taskInstance.getTaskCode(),
-            taskInstance.getState());
+                    taskInstance.getTaskCode(),
+                    taskInstance.getState());
 
         activeTaskProcessorMaps.remove(taskInstance.getTaskCode());
         stateWheelExecuteThread.removeTask4TimeoutCheck(processInstance, taskInstance);
@@ -503,7 +422,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      *
      * @param taskInstance
      */
-    private void releaseTaskGroup(TaskInstance taskInstance) {
+    public void releaseTaskGroup(TaskInstance taskInstance) {
         if (taskInstance.getTaskGroupId() > 0) {
             TaskInstance nextTaskInstance = this.processService.releaseTaskGroup(taskInstance);
             if (nextTaskInstance != null) {
@@ -528,13 +447,15 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      *
      * @param taskInstance
      */
-    private void retryTaskInstance(TaskInstance taskInstance) {
+    private void retryTaskInstance(TaskInstance taskInstance) throws StateEventHandleException {
         if (!taskInstance.taskCanRetry()) {
             return;
         }
         TaskInstance newTaskInstance = cloneRetryTaskInstance(taskInstance);
         if (newTaskInstance == null) {
-            logger.error("retry fail, new taskInstance is null, task code:{}, task id:{}", taskInstance.getTaskCode(), taskInstance.getId());
+            logger.error("retry fail, new taskInstance is null, task code:{}, task id:{}",
+                         taskInstance.getTaskCode(),
+                         taskInstance.getId());
             return;
         }
         waitToRetryTaskInstanceMap.put(newTaskInstance.getTaskCode(), newTaskInstance);
@@ -556,20 +477,6 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         }
     }
 
-    /**
-     * handle task retry event
-     *
-     * @param stateEvent
-     * @return
-     */
-    private boolean taskRetryEventHandler(StateEvent stateEvent) {
-        TaskInstance taskInstance = waitToRetryTaskInstanceMap.get(stateEvent.getTaskCode());
-        addTaskToStandByList(taskInstance);
-        submitStandByTask();
-        waitToRetryTaskInstanceMap.remove(stateEvent.getTaskCode());
-        return true;
-    }
-
     /**
      * update process instance
      */
@@ -603,43 +510,23 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     /**
      * check process instance by state event
      */
-    public boolean checkProcessInstance(StateEvent stateEvent) {
+    public void checkProcessInstance(StateEvent stateEvent) throws StateEventHandleError {
         if (this.processInstance.getId() != stateEvent.getProcessInstanceId()) {
-            logger.error("mismatch process instance id: {}, state event:{}", this.processInstance.getId(), stateEvent);
-            return false;
+            throw new StateEventHandleError("The event doesn't contains process instance id");
         }
-        return true;
     }
 
     /**
      * check if task instance exist by state event
      */
-    public boolean checkTaskInstanceByStateEvent(StateEvent stateEvent) {
+    public void checkTaskInstanceByStateEvent(StateEvent stateEvent) throws StateEventHandleError {
         if (stateEvent.getTaskInstanceId() == 0) {
-            logger.error("task instance id null, state event:{}", stateEvent);
-            return false;
+            throw new StateEventHandleError("The taskInstanceId is 0");
         }
 
         if (!taskInstanceMap.containsKey(stateEvent.getTaskInstanceId())) {
-            logger.error("mismatch task instance id, event:{}", stateEvent);
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * check if task instance exist by task code
-     */
-    public boolean checkTaskInstanceByCode(long taskCode) {
-        if (taskInstanceMap.isEmpty()) {
-            return false;
-        }
-        for (TaskInstance taskInstance : taskInstanceMap.values()) {
-            if (taskInstance.getTaskCode() == taskCode) {
-                return true;
-            }
+            throw new StateEventHandleError("Cannot find the taskInstance from taskInstanceMap");
         }
-        return false;
     }
 
     /**
@@ -688,59 +575,13 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         return Optional.empty();
     }
 
-    private boolean processStateChangeHandler(StateEvent stateEvent) {
-        try {
-            logger.info("process:{} state {} change to {}", processInstance.getId(), processInstance.getState(), stateEvent.getExecutionStatus());
-
-            if (stateEvent.getExecutionStatus() == ExecutionStatus.STOP) {
-                // serial wait execution type needs to wake up the waiting process
-                if (processDefinition.getExecutionType().typeIsSerialWait() || processDefinition.getExecutionType().typeIsSerialPriority()) {
-                    endProcess();
-                    return true;
-                }
-                this.updateProcessInstanceState(stateEvent);
-                return true;
-            }
-
-            if (processComplementData()) {
-                return true;
-            }
-            if (stateEvent.getExecutionStatus().typeIsFinished()) {
-                endProcess();
-            }
-            if (processInstance.getState() == ExecutionStatus.READY_STOP) {
-                killAllTasks();
-            }
-            return true;
-        } catch (Exception e) {
-            logger.error("process state change error:", e);
-        }
-        return true;
-    }
-
-    private boolean processBlockHandler(StateEvent stateEvent) {
-        try {
-            Optional<TaskInstance> taskInstanceOptional = getTaskInstance(stateEvent.getTaskInstanceId());
-            TaskInstance task = taskInstanceOptional.orElseThrow(
-                    () -> new RuntimeException("Cannot find taskInstance by taskInstanceId:" + stateEvent.getTaskInstanceId()));
-            if (!checkTaskInstanceByStateEvent(stateEvent)) {
-                logger.error("task {} is not a blocking task", task.getTaskCode());
-                return false;
-            }
-
-            BlockingParameters parameters = JSONUtils.parseObject(task.getTaskParams(), BlockingParameters.class);
-            if (parameters.isAlertWhenBlocking()) {
-                ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(processInstance.getId());
-                processAlertManager.sendProcessBlockingAlert(processInstance, projectUser);
-                logger.info("processInstance {} block alert send successful!", processInstance.getId());
-            }
-        } catch (Exception e) {
-            logger.error("sending blocking message error:", e);
-        }
-        return true;
+    public void processBlock() {
+        ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(processInstance.getId());
+        processAlertManager.sendProcessBlockingAlert(processInstance, projectUser);
+        logger.info("processInstance {} block alert send successful!", processInstance.getId());
     }
 
-    private boolean processComplementData() throws Exception {
+    public boolean processComplementData() {
         if (!needComplementProcess()) {
             return false;
         }
@@ -937,7 +778,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     /**
      * init task queue
      */
-    private void initTaskQueue() {
+    private void initTaskQueue() throws StateEventHandleException {
 
         taskFailedSubmit = false;
         activeTaskProcessorMaps.clear();
@@ -946,7 +787,8 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         errorTaskMap.clear();
 
         if (!isNewProcessInstance()) {
-            List<TaskInstance> validTaskInstanceList = processService.findValidTaskListByProcessId(processInstance.getId());
+            List<TaskInstance> validTaskInstanceList
+                = processService.findValidTaskListByProcessId(processInstance.getId());
             for (TaskInstance task : validTaskInstanceList) {
                 if (validTaskMap.containsKey(task.getTaskCode())) {
                     int oldTaskInstanceId = validTaskMap.get(task.getTaskCode());
@@ -956,7 +798,8 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
                         processService.updateTaskInstance(task);
                         continue;
                     }
-                    logger.warn("have same taskCode taskInstance when init task queue, taskCode:{}", task.getTaskCode());
+                    logger.warn("have same taskCode taskInstance when init task queue, taskCode:{}",
+                                task.getTaskCode());
                 }
 
                 validTaskMap.put(task.getTaskCode(), task.getId());
@@ -1099,6 +942,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             Host host = new Host(taskInstance.getHost());
             nettyExecutorManager.doExecute(host, hostUpdateCommand.convert2Command());
         } catch (Exception e) {
+            // Do we need to catch this exception?
             logger.error("notify process host update", e);
         }
     }
@@ -1348,8 +1192,11 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         return validTaskInstanceList;
     }
 
-    private void submitPostNode(String parentNodeCode) {
-        Set<String> submitTaskNodeList = DagHelper.parsePostNodes(parentNodeCode, skipTaskNodeMap, dag, getCompleteTaskInstanceMap());
+    private void submitPostNode(String parentNodeCode) throws StateEventHandleException {
+        Set<String> submitTaskNodeList = DagHelper.parsePostNodes(parentNodeCode,
+                                                                  skipTaskNodeMap,
+                                                                  dag,
+                                                                  getCompleteTaskInstanceMap());
         List<TaskInstance> taskInstances = new ArrayList<>();
         for (String taskNode : submitTaskNodeList) {
             TaskNode taskNodeObject = dag.getNode(taskNode);
@@ -1692,35 +1539,19 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             return true;
         }
 
-        try {
-            Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam());
-            Date endTime = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
-            return processInstance.getScheduleTime().equals(endTime);
-        } catch (Exception e) {
-            logger.error("complement end failed ", e);
-            return false;
-        }
+        Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam());
+        Date endTime = DateUtils.getScheduleDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
+        return processInstance.getScheduleTime().equals(endTime);
     }
 
     /**
      * updateProcessInstance process instance state
      * after each batch of tasks is executed, the status of the process instance is updated
      */
-    private void updateProcessInstanceState() {
+    private void updateProcessInstanceState() throws StateEventHandleException {
         ExecutionStatus state = getProcessInstanceState(processInstance);
         if (processInstance.getState() != state) {
-            logger.info("work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
-                        processInstance.getId(),
-                        processInstance.getName(),
-                        processInstance.getState(),
-                        state,
-                        processInstance.getCommandType());
-
-            processInstance.setState(state);
-            if (state.typeIsFinished()) {
-                processInstance.setEndTime(new Date());
-            }
-            processService.updateProcessInstance(processInstance);
+            updateWorkflowInstanceStatesToDB(state);
 
             StateEvent stateEvent = new StateEvent();
             stateEvent.setExecutionStatus(processInstance.getState());
@@ -1735,21 +1566,33 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     /**
      * stateEvent's execution status as process instance state
      */
-    private void updateProcessInstanceState(StateEvent stateEvent) {
+    public void updateProcessInstanceState(StateEvent stateEvent) throws StateEventHandleException {
         ExecutionStatus state = stateEvent.getExecutionStatus();
-        if (processInstance.getState() != state) {
+        updateWorkflowInstanceStatesToDB(state);
+    }
+
+    private void updateWorkflowInstanceStatesToDB(ExecutionStatus newStates) throws StateEventHandleException {
+        ExecutionStatus originStates = processInstance.getState();
+        if (originStates != newStates) {
             logger.info("work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
-                        processInstance.getId(),
-                        processInstance.getName(),
-                        processInstance.getState(),
-                        state,
-                        processInstance.getCommandType());
-
-            processInstance.setState(state);
-            if (state.typeIsFinished()) {
+                processInstance.getId(),
+                processInstance.getName(),
+                originStates,
+                newStates,
+                processInstance.getCommandType());
+
+            processInstance.setState(newStates);
+            if (newStates.typeIsFinished()) {
                 processInstance.setEndTime(new Date());
             }
-            processService.updateProcessInstance(processInstance);
+            try {
+                processService.updateProcessInstance(processInstance);
+            } catch (Exception ex) {
+                // recover the status
+                processInstance.setState(originStates);
+                processInstance.setEndTime(null);
+                throw new StateEventHandleException("Update process instance status to DB error", ex);
+            }
         }
     }
 
@@ -1768,18 +1611,17 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      *
      * @param taskInstance task instance
      */
-    private void addTaskToStandByList(TaskInstance taskInstance) {
-        try {
-            if (readyToSubmitTaskQueue.contains(taskInstance)) {
-                logger.warn("task was found in ready submit queue, task code:{}", taskInstance.getTaskCode());
-                return;
-            }
-            logger.info("add task to stand by list, task name:{}, task id:{}, task code:{}",
-                taskInstance.getName(), taskInstance.getId(), taskInstance.getTaskCode());
-            readyToSubmitTaskQueue.put(taskInstance);
-        } catch (Exception e) {
-            logger.error("add task instance to readyToSubmitTaskQueue, taskName:{}, task id:{}", taskInstance.getName(), taskInstance.getId(), e);
+    public void addTaskToStandByList(TaskInstance taskInstance) {
+        if (readyToSubmitTaskQueue.contains(taskInstance)) {
+            logger.warn("task was found in ready submit queue, task code:{}", taskInstance.getTaskCode());
+            return;
         }
+        logger.info("add task to stand by list, task name:{}, task id:{}, task code:{}",
+            taskInstance.getName(),
+            taskInstance.getId(),
+            taskInstance.getTaskCode());
+        TaskMetrics.incTaskSubmit();
+        readyToSubmitTaskQueue.put(taskInstance);
     }
 
     /**
@@ -1788,15 +1630,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      * @param taskInstance task instance
      */
     private void removeTaskFromStandbyList(TaskInstance taskInstance) {
-        logger.info("remove task from stand by list, id: {} name:{}", taskInstance.getId(), taskInstance.getName());
-        try {
-            readyToSubmitTaskQueue.remove(taskInstance);
-        } catch (Exception e) {
-            logger.error("remove task instance from readyToSubmitTaskQueue error, task id:{}, Name: {}",
-                         taskInstance.getId(),
-                         taskInstance.getName(),
-                         e);
-        }
+        readyToSubmitTaskQueue.remove(taskInstance);
     }
 
     /**
@@ -1816,10 +1650,10 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     /**
      * close the on going tasks
      */
-    private void killAllTasks() {
+    public void killAllTasks() {
         logger.info("kill called on process instance id: {}, num: {}",
-                    processInstance.getId(),
-                    activeTaskProcessorMaps.size());
+            processInstance.getId(),
+            activeTaskProcessorMaps.size());
 
         if (readyToSubmitTaskQueue.size() > 0) {
             readyToSubmitTaskQueue.clear();
@@ -1854,7 +1688,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     /**
      * handling the list of tasks to be submitted
      */
-    private void submitStandByTask() {
+    public void submitStandByTask() throws StateEventHandleException {
         int length = readyToSubmitTaskQueue.size();
         for (int i = 0; i < length; i++) {
             TaskInstance task = readyToSubmitTaskQueue.peek();
@@ -2013,6 +1847,19 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         }
     }
 
+    public Map<Long, Integer> getCompleteTaskMap() {
+        return completeTaskMap;
+    }
+
+    public Map<Long, ITaskProcessor> getActiveTaskProcessMap() {
+        return activeTaskProcessorMaps;
+    }
+
+    public Map<Long, TaskInstance> getWaitToRetryTaskInstanceMap() {
+        return waitToRetryTaskInstanceMap;
+    }
+
+
     private void setGlobalParamIfCommanded(ProcessDefinition processDefinition, Map<String, String> cmdParam) {
         // get start params from command param
         Map<String, String> startParamMap = new HashMap<>();
@@ -2047,46 +1894,4 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         }
     }
 
-    private void measureProcessState(StateEvent processStateEvent) {
-        if (processStateEvent.getExecutionStatus().typeIsFinished()) {
-            ProcessInstanceMetrics.incProcessInstanceFinish();
-        }
-        switch (processStateEvent.getExecutionStatus()) {
-            case STOP:
-                ProcessInstanceMetrics.incProcessInstanceStop();
-                break;
-            case SUCCESS:
-                ProcessInstanceMetrics.incProcessInstanceSuccess();
-                break;
-            case FAILURE:
-                ProcessInstanceMetrics.incProcessInstanceFailure();
-                break;
-            default:
-                break;
-        }
-    }
-
-    private void measureTaskState(StateEvent taskStateEvent) {
-        if (taskStateEvent == null || taskStateEvent.getExecutionStatus() == null) {
-            // the event is broken
-            logger.warn("The task event is broken..., taskEvent: {}", taskStateEvent);
-            return;
-        }
-        if (taskStateEvent.getExecutionStatus().typeIsFinished()) {
-            TaskMetrics.incTaskFinish();
-        }
-        switch (taskStateEvent.getExecutionStatus()) {
-            case STOP:
-                TaskMetrics.incTaskStop();
-                break;
-            case SUCCESS:
-                TaskMetrics.incTaskSuccess();
-                break;
-            case FAILURE:
-                TaskMetrics.incTaskFailure();
-                break;
-            default:
-                break;
-        }
-    }
 }
\ No newline at end of file
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index ba097109be..bb49d9dea1 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -18,7 +18,7 @@
 package org.apache.dolphinscheduler.server.master.runner;
 
 import org.apache.dolphinscheduler.common.enums.Flag;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
@@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
 import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
 import org.apache.commons.lang.StringUtils;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
index 663cba4d90..313a43c0e0 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
@@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.master.service;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.model.Server;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
index fc9bcca3aa..98bf514730 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
@@ -26,7 +26,7 @@ import static org.mockito.Mockito.doNothing;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.model.Server;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java
index 247e4066f8..0d623e398a 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/NettyRemoteChannel.java
@@ -17,12 +17,13 @@
 
 package org.apache.dolphinscheduler.remote.processor;
 
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelFuture;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
 import org.apache.dolphinscheduler.remote.utils.Host;
 
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+
 /**
  *  callback channel
  */
@@ -72,7 +73,7 @@ public class NettyRemoteChannel {
         return this.channel.isActive();
     }
 
-    public ChannelFuture writeAndFlush(Command command){
+    public ChannelFuture writeAndFlush(Command command) {
         return this.channel.writeAndFlush(command);
     }
 
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java
index af51831068..be564261fb 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java
@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
 import org.apache.dolphinscheduler.remote.utils.Host;
 
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.slf4j.Logger;
@@ -71,19 +72,19 @@ public class StateEventCallbackService {
      * @param host
      * @return callback channel
      */
-    private NettyRemoteChannel newRemoteChannel(Host host) {
+    private Optional<NettyRemoteChannel> newRemoteChannel(Host host) {
         Channel newChannel;
         NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(host.getAddress());
         if (nettyRemoteChannel != null) {
             if (nettyRemoteChannel.isActive()) {
-                return nettyRemoteChannel;
+                return Optional.of(nettyRemoteChannel);
             }
         }
         newChannel = nettyRemotingClient.getChannel(host);
         if (newChannel != null) {
-            return newRemoteChannel(newChannel, host.getAddress());
+            return Optional.of(newRemoteChannel(newChannel, host.getAddress()));
         }
-        return null;
+        return Optional.empty();
     }
 
     public long pause(int ntries) {
@@ -110,16 +111,26 @@ public class StateEventCallbackService {
     }
 
     /**
-     * send result
+     * Send the command to target address, this method doesn't guarantee the command send success.
      *
-     * @param command command
+     * @param command command need tp send
      */
     public void sendResult(String address, int port, Command command) {
         logger.info("send result, host:{}, command:{}", address, command.toString());
         Host host = new Host(address, port);
-        NettyRemoteChannel nettyRemoteChannel = newRemoteChannel(host);
-        if (nettyRemoteChannel != null) {
+        sendResult(host, command);
+    }
+
+    /**
+     * Send the command to target host, this method doesn't guarantee the command send success.
+     *
+     * @param host    target host
+     * @param command command need to send
+     */
+    public void sendResult(Host host, Command command) {
+        logger.info("send result, host:{}, command:{}", host.getAddress(), command.toString());
+        newRemoteChannel(host).ifPresent(nettyRemoteChannel -> {
             nettyRemoteChannel.writeAndFlush(command);
-        }
+        });
     }
 }
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
index cc7f39e402..0cf03abe8d 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
@@ -51,10 +51,9 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      * put task instance to priority queue
      *
      * @param taskInstance taskInstance
-     * @throws TaskPriorityQueueException
      */
     @Override
-    public void put(TaskInstance taskInstance) throws TaskPriorityQueueException {
+    public void put(TaskInstance taskInstance) {
         Preconditions.checkNotNull(taskInstance);
         queue.add(taskInstance);
         taskInstanceIdentifySet.add(getTaskInstanceIdentify(taskInstance));
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java
index 1325afe9b2..0007581d3c 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueue.java
@@ -33,7 +33,7 @@ public interface TaskPriorityQueue<T> {
      * @param taskInfo taskInfo
      * @throws TaskPriorityQueueException
      */
-    void put(T taskInfo) throws TaskPriorityQueueException;
+    void put(T taskInfo);
 
     /**
      * take taskInfo
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java
index fef5f8ff79..0d0eebe03d 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java
@@ -40,10 +40,9 @@ public class TaskPriorityQueueImpl implements TaskPriorityQueue<TaskPriority> {
      * put task takePriorityInfo
      *
      * @param taskPriorityInfo takePriorityInfo
-     * @throws TaskPriorityQueueException
      */
     @Override
-    public void put(TaskPriority taskPriorityInfo) throws TaskPriorityQueueException {
+    public void put(TaskPriority taskPriorityInfo) {
         queue.put(taskPriorityInfo);
     }
 
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java
index e080842c34..b62770984a 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java
@@ -58,7 +58,13 @@ public class TaskExecuteResponseAckProcessor implements NettyRequestProcessor {
         if (taskExecuteResponseAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()) {
             ResponseCache.get().removeResponseCache(taskExecuteResponseAckCommand.getTaskInstanceId());
             TaskCallbackService.remove(taskExecuteResponseAckCommand.getTaskInstanceId());
-            logger.debug("remove REMOTE_CHANNELS, task instance id:{}", taskExecuteResponseAckCommand.getTaskInstanceId());
+            logger.debug("remove REMOTE_CHANNELS, task instance id:{}",
+                taskExecuteResponseAckCommand.getTaskInstanceId());
+        } else if (taskExecuteResponseAckCommand.getStatus() == ExecutionStatus.FAILURE.getCode()) {
+            // master handle worker response error, will still retry
+        } else {
+            throw new IllegalArgumentException("Invalid task execute response ack status: "
+                + taskExecuteResponseAckCommand.getStatus());
         }
     }
 


[dolphinscheduler] 16/29: [Fix-10666] Workflow submit failed will still in memory and never retry (#10667)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 3b923e5933bf3f97d73849feef89e535635e4366
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Mon Jul 4 22:08:15 2022 +0800

    [Fix-10666] Workflow submit failed will still in memory and never retry (#10667)
    
    * Workflow submit failed will still in memory and never retry
    
    (cherry picked from commit 35a10d092f566c07137da5fb67b21cee644cdc8f)
---
 .../server/master/exception/MasterException.java   |  29 +++
 .../master/metrics/ProcessInstanceMetrics.java     |   6 +
 .../master/runner/MasterSchedulerService.java      | 159 ++++++++++------
 .../master/runner/StateWheelExecuteThread.java     |  18 +-
 .../master/runner/WorkflowExecuteRunnable.java     | 199 ++++++++++-----------
 .../server/master/runner/WorkflowSubmitStatue.java |  34 ++++
 .../WorkflowExecuteRunnableTest.java}              |  19 +-
 .../service/process/ProcessService.java            |   2 +-
 8 files changed, 296 insertions(+), 170 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/exception/MasterException.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/exception/MasterException.java
new file mode 100644
index 0000000000..eb5d5e8dfe
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/exception/MasterException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.exception;
+
+public class MasterException extends Exception {
+
+    public MasterException(String message) {
+        super(message);
+    }
+
+    public MasterException(String message, Throwable throwable) {
+        super(message, throwable);
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
index 957a3ecb7b..2845d2a637 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
@@ -90,6 +90,12 @@ public final class ProcessInstanceMetrics {
             .register(Metrics.globalRegistry);
     }
 
+    public static synchronized void registerProcessInstanceResubmitGauge(Supplier<Number> function) {
+        Gauge.builder("ds.workflow.instance.resubmit", function)
+            .description("The current process instance need to resubmit count")
+            .register(Metrics.globalRegistry);
+    }
+
     public static void incProcessInstanceSubmit() {
         PROCESS_INSTANCE_SUBMIT_COUNTER.increment();
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index 1d0071a4dd..ae7b4bdeb0 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -27,11 +27,10 @@ import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.dao.entity.Command;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.remote.NettyRemotingClient;
-import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
+import org.apache.dolphinscheduler.server.master.exception.MasterException;
 import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
 import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
@@ -43,7 +42,9 @@ import org.apache.commons.collections4.CollectionUtils;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 
 import org.slf4j.Logger;
@@ -51,6 +52,8 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import lombok.NonNull;
+
 /**
  * Master scheduler thread, this thread will consume the commands from database and trigger processInstance executed.
  */
@@ -71,8 +74,6 @@ public class MasterSchedulerService extends BaseDaemonThread {
     @Autowired
     private ProcessAlertManager processAlertManager;
 
-    private NettyRemotingClient nettyRemotingClient;
-
     @Autowired
     private NettyExecutorManager nettyExecutorManager;
 
@@ -93,6 +94,10 @@ public class MasterSchedulerService extends BaseDaemonThread {
     @Autowired
     private StateWheelExecuteThread stateWheelExecuteThread;
 
+    private final LinkedBlockingQueue<ProcessInstance> submitFailedProcessInstances = new LinkedBlockingQueue<>();
+
+    private Thread failedProcessInstanceResubmitThread;
+
     private String masterAddress;
 
     protected MasterSchedulerService() {
@@ -104,22 +109,23 @@ public class MasterSchedulerService extends BaseDaemonThread {
      */
     public void init() {
         this.masterPrepareExecService = (ThreadPoolExecutor) ThreadUtils.newDaemonFixedThreadExecutor("MasterPreExecThread", masterConfig.getPreExecThreads());
-        NettyClientConfig clientConfig = new NettyClientConfig();
-        this.nettyRemotingClient = new NettyRemotingClient(clientConfig);
         this.masterAddress = NetUtils.getAddr(masterConfig.getListenPort());
+        this.failedProcessInstanceResubmitThread = new FailedProcessInstanceResubmitThread(submitFailedProcessInstances);
+        ProcessInstanceMetrics.registerProcessInstanceResubmitGauge(submitFailedProcessInstances::size);
     }
 
     @Override
     public synchronized void start() {
         logger.info("Master schedule service starting..");
-        this.stateWheelExecuteThread.start();
         super.start();
+        this.failedProcessInstanceResubmitThread.start();
         logger.info("Master schedule service started...");
     }
 
     public void close() {
         logger.info("Master schedule service stopping...");
-        nettyRemotingClient.close();
+        // these process instances will be failover, so we can safa clear here
+        submitFailedProcessInstances.clear();
         logger.info("Master schedule service stopped...");
     }
 
@@ -142,7 +148,9 @@ public class MasterSchedulerService extends BaseDaemonThread {
                 Thread.currentThread().interrupt();
                 break;
             } catch (Exception e) {
-                logger.error("Master schedule service loop command error", e);
+                logger.error("Master schedule workflow error", e);
+                // sleep for 1s here to avoid the database down cause the exception boom
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
             }
         }
     }
@@ -150,7 +158,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
     /**
      * Query command from database by slot, and transform to workflow instance, then submit to workflowExecuteThreadPool.
      */
-    private void scheduleWorkflow() throws InterruptedException {
+    private void scheduleWorkflow() throws InterruptedException, MasterException {
         List<Command> commands = findCommands();
         if (CollectionUtils.isEmpty(commands)) {
             // indicate that no command ,sleep for 1s
@@ -160,37 +168,52 @@ public class MasterSchedulerService extends BaseDaemonThread {
 
         List<ProcessInstance> processInstances = command2ProcessInstance(commands);
         if (CollectionUtils.isEmpty(processInstances)) {
+            // indicate that the command transform to processInstance error, sleep for 1s
+            Thread.sleep(Constants.SLEEP_TIME_MILLIS);
             return;
         }
         MasterServerMetrics.incMasterConsumeCommand(commands.size());
 
         for (ProcessInstance processInstance : processInstances) {
-            try {
-                LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
-                logger.info("Master schedule service starting workflow instance");
-                final WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
-                    processInstance
-                    , processService
-                    , nettyExecutorManager
-                    , processAlertManager
-                    , masterConfig
-                    , stateWheelExecuteThread);
-
-                this.processInstanceExecCacheManager.cache(processInstance.getId(), workflowExecuteRunnable);
-                if (processInstance.getTimeout() > 0) {
-                    stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
-                }
-                ProcessInstanceMetrics.incProcessInstanceSubmit();
-                workflowExecuteThreadPool.submit(workflowExecuteRunnable);
-                logger.info("Master schedule service started workflow instance");
-
-            } catch (Exception ex) {
-                processInstanceExecCacheManager.removeByProcessInstanceId(processInstance.getId());
-                stateWheelExecuteThread.removeProcess4TimeoutCheck(processInstance.getId());
-                logger.info("Master submit workflow to thread pool failed, will remove workflow runnable from cache manager", ex);
-            } finally {
-                LoggerUtils.removeWorkflowInstanceIdMDC();
+            submitProcessInstance(processInstance);
+        }
+    }
+
+    private void submitProcessInstance(@NonNull ProcessInstance processInstance) {
+        try {
+            LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
+            logger.info("Master schedule service starting workflow instance");
+            final WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
+                processInstance
+                , processService
+                , nettyExecutorManager
+                , processAlertManager
+                , masterConfig
+                , stateWheelExecuteThread);
+
+            this.processInstanceExecCacheManager.cache(processInstance.getId(), workflowExecuteRunnable);
+            if (processInstance.getTimeout() > 0) {
+                stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
             }
+            ProcessInstanceMetrics.incProcessInstanceSubmit();
+            CompletableFuture<WorkflowSubmitStatue> workflowSubmitFuture = CompletableFuture.supplyAsync(
+                workflowExecuteRunnable::call, workflowExecuteThreadPool);
+            workflowSubmitFuture.thenAccept(workflowSubmitStatue -> {
+                if (WorkflowSubmitStatue.FAILED == workflowSubmitStatue) {
+                    // submit failed
+                    processInstanceExecCacheManager.removeByProcessInstanceId(processInstance.getId());
+                    stateWheelExecuteThread.removeProcess4TimeoutCheck(processInstance.getId());
+                    submitFailedProcessInstances.add(processInstance);
+                }
+            });
+            logger.info("Master schedule service started workflow instance");
+
+        } catch (Exception ex) {
+            processInstanceExecCacheManager.removeByProcessInstanceId(processInstance.getId());
+            stateWheelExecuteThread.removeProcess4TimeoutCheck(processInstance.getId());
+            logger.info("Master submit workflow to thread pool failed, will remove workflow runnable from cache manager", ex);
+        } finally {
+            LoggerUtils.removeWorkflowInstanceIdMDC();
         }
     }
 
@@ -232,23 +255,27 @@ public class MasterSchedulerService extends BaseDaemonThread {
         return processInstances;
     }
 
-    private List<Command> findCommands() {
-        long scheduleStartTime = System.currentTimeMillis();
-        int thisMasterSlot = ServerNodeManager.getSlot();
-        int masterCount = ServerNodeManager.getMasterSize();
-        if (masterCount <= 0) {
-            logger.warn("Master count: {} is invalid, the current slot: {}", masterCount, thisMasterSlot);
-            return Collections.emptyList();
-        }
-        int pageNumber = 0;
-        int pageSize = masterConfig.getFetchCommandNum();
-        final List<Command> result = processService.findCommandPageBySlot(pageSize, pageNumber, masterCount, thisMasterSlot);
-        if (CollectionUtils.isNotEmpty(result)) {
-            logger.info("Master schedule service loop command success, command size: {}, current slot: {}, total slot size: {}",
-                result.size(), thisMasterSlot, masterCount);
+    private List<Command> findCommands() throws MasterException {
+        try {
+            long scheduleStartTime = System.currentTimeMillis();
+            int thisMasterSlot = ServerNodeManager.getSlot();
+            int masterCount = ServerNodeManager.getMasterSize();
+            if (masterCount <= 0) {
+                logger.warn("Master count: {} is invalid, the current slot: {}", masterCount, thisMasterSlot);
+                return Collections.emptyList();
+            }
+            int pageNumber = 0;
+            int pageSize = masterConfig.getFetchCommandNum();
+            final List<Command> result = processService.findCommandPageBySlot(pageSize, pageNumber, masterCount, thisMasterSlot);
+            if (CollectionUtils.isNotEmpty(result)) {
+                logger.info("Master schedule service loop command success, command size: {}, current slot: {}, total slot size: {}",
+                    result.size(), thisMasterSlot, masterCount);
+            }
+            ProcessInstanceMetrics.recordCommandQueryTime(System.currentTimeMillis() - scheduleStartTime);
+            return result;
+        } catch (Exception ex) {
+            throw new MasterException("Master loop command from database error", ex);
         }
-        ProcessInstanceMetrics.recordCommandQueryTime(System.currentTimeMillis() - scheduleStartTime);
-        return result;
     }
 
     private SlotCheckState slotCheck(Command command) {
@@ -265,4 +292,34 @@ public class MasterSchedulerService extends BaseDaemonThread {
         return state;
     }
 
+    private class FailedProcessInstanceResubmitThread extends Thread {
+
+        private final LinkedBlockingQueue<ProcessInstance> submitFailedProcessInstances;
+
+        public FailedProcessInstanceResubmitThread(LinkedBlockingQueue<ProcessInstance> submitFailedProcessInstances) {
+            logger.info("Starting workflow resubmit thread");
+            this.submitFailedProcessInstances = submitFailedProcessInstances;
+            this.setDaemon(true);
+            this.setName("SubmitFailedProcessInstanceHandleThread");
+            logger.info("Started workflow resubmit thread");
+        }
+
+        @Override
+        public void run() {
+            while (Stopper.isRunning()) {
+                try {
+                    ProcessInstance processInstance = submitFailedProcessInstances.take();
+                    submitProcessInstance(processInstance);
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    logger.warn("SubmitFailedProcessInstanceHandleThread has been interrupted, will return");
+                    break;
+                }
+
+                // avoid the failed-fast cause CPU higher
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
+            }
+        }
+    }
+
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 8b92696723..a56a7d8c5a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -33,12 +33,11 @@ import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheM
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
 
-import org.apache.commons.lang3.ThreadUtils;
-
-import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
+import javax.annotation.PostConstruct;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -91,9 +90,14 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
         super("StateWheelExecuteThread");
     }
 
+    @PostConstruct
+    public void startWheelThread() {
+        super.start();
+    }
+
     @Override
     public void run() {
-        Duration checkInterval = masterConfig.getStateWheelInterval();
+        final long checkInterval = masterConfig.getStateWheelInterval().toMillis();
         while (Stopper.isRunning()) {
             try {
                 checkTask4Timeout();
@@ -104,9 +108,11 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
                 logger.error("state wheel thread check error:", e);
             }
             try {
-                ThreadUtils.sleep(checkInterval);
+                Thread.sleep(checkInterval);
             } catch (InterruptedException e) {
-                logger.error("state wheel thread sleep error", e);
+                logger.error("state wheel thread sleep error, will close the loop", e);
+                Thread.currentThread().interrupt();
+                break;
             }
         }
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 7676367a18..88d7391dde 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -81,12 +81,14 @@ import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
 import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
 
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.math.NumberUtils;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -96,30 +98,29 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Lists;
 
+import lombok.NonNull;
+
 /**
  * Workflow execute task, used to execute a workflow instance.
  */
-public class WorkflowExecuteRunnable implements Runnable {
+public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
 
     /**
      * logger of WorkflowExecuteThread
      */
     private static final Logger logger = LoggerFactory.getLogger(WorkflowExecuteRunnable.class);
 
-    /**
-     * master config
-     */
-    private final MasterConfig masterConfig;
-
     /**
      * process service
      */
@@ -151,14 +152,14 @@ public class WorkflowExecuteRunnable implements Runnable {
     private DAG<String, TaskNode, TaskNodeRelation> dag;
 
     /**
-     * key of workflow
+     * unique key of workflow
      */
     private String key;
 
     /**
      * start flag, true: start nodes submit completely
      */
-    private boolean isStart = false;
+    private volatile boolean isStart = false;
 
     /**
      * submit failure nodes
@@ -235,6 +236,8 @@ public class WorkflowExecuteRunnable implements Runnable {
      */
     private final StateWheelExecuteThread stateWheelExecuteThread;
 
+    private final String masterAddress;
+
     /**
      * @param processInstance         processInstance
      * @param processService          processService
@@ -243,19 +246,19 @@ public class WorkflowExecuteRunnable implements Runnable {
      * @param masterConfig            masterConfig
      * @param stateWheelExecuteThread stateWheelExecuteThread
      */
-    public WorkflowExecuteRunnable(ProcessInstance processInstance
-            , ProcessService processService
-            , NettyExecutorManager nettyExecutorManager
-            , ProcessAlertManager processAlertManager
-            , MasterConfig masterConfig
-            , StateWheelExecuteThread stateWheelExecuteThread) {
+    public WorkflowExecuteRunnable(@NonNull ProcessInstance processInstance,
+                                   @NonNull ProcessService processService,
+                                   @NonNull NettyExecutorManager nettyExecutorManager,
+                                   @NonNull ProcessAlertManager processAlertManager,
+                                   @NonNull MasterConfig masterConfig,
+                                   @NonNull StateWheelExecuteThread stateWheelExecuteThread) {
         this.processService = processService;
         this.processInstance = processInstance;
-        this.masterConfig = masterConfig;
         this.nettyExecutorManager = nettyExecutorManager;
         this.processAlertManager = processAlertManager;
         this.stateWheelExecuteThread = stateWheelExecuteThread;
-        TaskMetrics.registerTaskRunning(readyToSubmitTaskQueue::size);
+        this.masterAddress = NetUtils.getAddr(masterConfig.getListenPort());
+        TaskMetrics.registerTaskPrepared(readyToSubmitTaskQueue::size);
     }
 
     /**
@@ -280,6 +283,7 @@ public class WorkflowExecuteRunnable implements Runnable {
                     this.stateEvents.remove(stateEvent);
                 }
             } catch (Exception e) {
+                // we catch the exception here, since if the state event handle failed, the state event will still keep in the stateEvents queue.
                 logger.error("state handle error:", e);
             } finally {
                 LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
@@ -464,6 +468,7 @@ public class WorkflowExecuteRunnable implements Runnable {
 
         if (taskInstance.getState().typeIsSuccess()) {
             completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
+            // todo: merge the last taskInstance
             processInstance.setVarPool(taskInstance.getVarPool());
             processService.saveProcessInstance(processInstance);
             if (!processInstance.isBlocked()) {
@@ -822,18 +827,24 @@ public class WorkflowExecuteRunnable implements Runnable {
      * ProcessInstance start entrypoint.
      */
     @Override
-    public void run() {
+    public WorkflowSubmitStatue call() {
         if (this.taskInstanceMap.size() > 0 || isStart) {
             logger.warn("The workflow has already been started");
-            return;
+            return WorkflowSubmitStatue.DUPLICATED_SUBMITTED;
         }
+
         try {
+            LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
             buildFlowDag();
             initTaskQueue();
             submitPostNode(null);
             isStart = true;
+            return WorkflowSubmitStatue.SUCCESS;
         } catch (Exception e) {
-            logger.error("start process error, process instance id:{}", processInstance.getId(), e);
+            logger.error("Start workflow error", e);
+            return WorkflowSubmitStatue.FAILED;
+        } finally {
+            LoggerUtils.removeWorkflowInstanceIdMDC();
         }
     }
 
@@ -883,7 +894,7 @@ public class WorkflowExecuteRunnable implements Runnable {
     }
 
     /**
-     * generate process dag
+     * Generate process dag
      *
      * @throws Exception exception
      */
@@ -895,7 +906,7 @@ public class WorkflowExecuteRunnable implements Runnable {
                                                                  processInstance.getProcessDefinitionVersion());
         processInstance.setProcessDefinition(processDefinition);
 
-        List<TaskInstance> recoverNodeList = getStartTaskInstanceList(processInstance.getCommandParam());
+        List<TaskInstance> recoverNodeList = getRecoverTaskInstanceList(processInstance.getCommandParam());
 
         List<ProcessTaskRelation> processTaskRelations = processService.findRelationByCode(processDefinition.getCode(), processDefinition.getVersion());
         List<TaskDefinitionLog> taskDefinitionLogs = processService.getTaskDefineLogListByRelation(processTaskRelations);
@@ -990,7 +1001,8 @@ public class WorkflowExecuteRunnable implements Runnable {
                 if (complementListDate.isEmpty() && needComplementProcess()) {
                     complementListDate = CronUtils.getSelfFireDateList(start, end, schedules);
                     logger.info(" process definition code:{} complement data: {}",
-                        processInstance.getProcessDefinitionCode(), complementListDate.toString());
+                                processInstance.getProcessDefinitionCode(),
+                                complementListDate.toString());
 
                     if (!complementListDate.isEmpty() && Flag.NO == processInstance.getIsSubProcess()) {
                         processInstance.setScheduleTime(complementListDate.get(0));
@@ -1082,7 +1094,7 @@ public class WorkflowExecuteRunnable implements Runnable {
 
         try {
             HostUpdateCommand hostUpdateCommand = new HostUpdateCommand();
-            hostUpdateCommand.setProcessHost(NetUtils.getAddr(masterConfig.getListenPort()));
+            hostUpdateCommand.setProcessHost(masterAddress);
             hostUpdateCommand.setTaskInstanceId(taskInstance.getId());
             Host host = new Host(taskInstance.getHost());
             nettyExecutorManager.doExecute(host, hostUpdateCommand.convert2Command());
@@ -1843,105 +1855,84 @@ public class WorkflowExecuteRunnable implements Runnable {
      * handling the list of tasks to be submitted
      */
     private void submitStandByTask() {
-        try {
-            int length = readyToSubmitTaskQueue.size();
-            for (int i = 0; i < length; i++) {
-                TaskInstance task = readyToSubmitTaskQueue.peek();
-                if (task == null) {
+        int length = readyToSubmitTaskQueue.size();
+        for (int i = 0; i < length; i++) {
+            TaskInstance task = readyToSubmitTaskQueue.peek();
+            if (task == null) {
+                continue;
+            }
+            // stop tasks which is retrying if forced success happens
+            if (task.taskCanRetry()) {
+                TaskInstance retryTask = processService.findTaskInstanceById(task.getId());
+                if (retryTask != null && retryTask.getState().equals(ExecutionStatus.FORCED_SUCCESS)) {
+                    task.setState(retryTask.getState());
+                    logger.info("task: {} has been forced success, put it into complete task list and stop retrying",
+                                task.getName());
+                    removeTaskFromStandbyList(task);
+                    completeTaskMap.put(task.getTaskCode(), task.getId());
+                    taskInstanceMap.put(task.getId(), task);
+                    submitPostNode(Long.toString(task.getTaskCode()));
                     continue;
                 }
-                // stop tasks which is retrying if forced success happens
-                if (task.taskCanRetry()) {
-                    TaskInstance retryTask = processService.findTaskInstanceById(task.getId());
-                    if (retryTask != null && retryTask.getState().equals(ExecutionStatus.FORCED_SUCCESS)) {
-                        task.setState(retryTask.getState());
-                        logger.info("task: {} has been forced success, put it into complete task list and stop retrying", task.getName());
-                        removeTaskFromStandbyList(task);
-                        completeTaskMap.put(task.getTaskCode(), task.getId());
-                        taskInstanceMap.put(task.getId(), task);
-                        submitPostNode(Long.toString(task.getTaskCode()));
-                        continue;
-                    }
-                }
-                //init varPool only this task is the first time running
-                if (task.isFirstRun()) {
-                    //get pre task ,get all the task varPool to this task
-                    Set<String> preTask = dag.getPreviousNodes(Long.toString(task.getTaskCode()));
-                    getPreVarPool(task, preTask);
-                }
-                DependResult dependResult = getDependResultForTask(task);
-                if (DependResult.SUCCESS == dependResult) {
-                    Optional<TaskInstance> taskInstanceOptional = submitTaskExec(task);
-                    if (!taskInstanceOptional.isPresent()) {
-                        this.taskFailedSubmit = true;
-                        // Remove and add to complete map and error map
-                        removeTaskFromStandbyList(task);
-                        completeTaskMap.put(task.getTaskCode(), task.getId());
-                        errorTaskMap.put(task.getTaskCode(), task.getId());
-                        logger.error("Task submitted failed, processInstanceId: {}, taskInstanceId: {}",
-                                     task.getProcessInstanceId(),
-                                     task.getId());
-                    } else {
-                        removeTaskFromStandbyList(task);
-                    }
-                } else if (DependResult.FAILED == dependResult) {
-                    // if the dependency fails, the current node is not submitted and the state changes to failure.
-                    dependFailedTaskMap.put(task.getTaskCode(), task.getId());
+            }
+            //init varPool only this task is the first time running
+            if (task.isFirstRun()) {
+                //get pre task ,get all the task varPool to this task
+                Set<String> preTask = dag.getPreviousNodes(Long.toString(task.getTaskCode()));
+                getPreVarPool(task, preTask);
+            }
+            DependResult dependResult = getDependResultForTask(task);
+            if (DependResult.SUCCESS == dependResult) {
+                Optional<TaskInstance> taskInstanceOptional = submitTaskExec(task);
+                if (!taskInstanceOptional.isPresent()) {
+                    this.taskFailedSubmit = true;
+                    // Remove and add to complete map and error map
                     removeTaskFromStandbyList(task);
-                    logger.info("Task dependent result is failed, taskInstanceId:{} depend result : {}",
-                                task.getId(),
-                                dependResult);
-                } else if (DependResult.NON_EXEC == dependResult) {
-                    // for some reasons(depend task pause/stop) this task would not be submit
+                    completeTaskMap.put(task.getTaskCode(), task.getId());
+                    errorTaskMap.put(task.getTaskCode(), task.getId());
+                    logger.error("Task submitted failed, processInstanceId: {}, taskInstanceId: {}",
+                                 task.getProcessInstanceId(),
+                                 task.getId());
+                } else {
                     removeTaskFromStandbyList(task);
-                    logger.info("Remove task due to depend result not executed, taskInstanceId:{} depend result : {}",
-                                task.getId(),
-                                dependResult);
                 }
+            } else if (DependResult.FAILED == dependResult) {
+                // if the dependency fails, the current node is not submitted and the state changes to failure.
+                dependFailedTaskMap.put(task.getTaskCode(), task.getId());
+                removeTaskFromStandbyList(task);
+                logger.info("Task dependent result is failed, taskInstanceId:{} depend result : {}",
+                            task.getId(),
+                            dependResult);
+            } else if (DependResult.NON_EXEC == dependResult) {
+                // for some reasons(depend task pause/stop) this task would not be submit
+                removeTaskFromStandbyList(task);
+                logger.info("Remove task due to depend result not executed, taskInstanceId:{} depend result : {}",
+                            task.getId(),
+                            dependResult);
             }
-        } catch (Exception e) {
-            logger.error("submit standby task error", e);
         }
     }
 
     /**
-     * get recovery task instance list
-     *
-     * @param taskIdArray task id array
-     * @return recovery task instance list
-     */
-    private List<TaskInstance> getRecoverTaskInstanceList(String[] taskIdArray) {
-        if (taskIdArray == null || taskIdArray.length == 0) {
-            return new ArrayList<>();
-        }
-        List<Integer> taskIdList = new ArrayList<>(taskIdArray.length);
-        for (String taskId : taskIdArray) {
-            try {
-                Integer id = Integer.valueOf(taskId);
-                taskIdList.add(id);
-            } catch (Exception e) {
-                logger.error("get recovery task instance failed ", e);
-            }
-        }
-        return processService.findTaskInstanceByIdList(taskIdList);
-    }
-
-    /**
-     * get start task instance list
+     * Get start task instance list from recover
      *
      * @param cmdParam command param
      * @return task instance list
      */
-    private List<TaskInstance> getStartTaskInstanceList(String cmdParam) {
-
-        List<TaskInstance> instanceList = new ArrayList<>();
+    protected List<TaskInstance> getRecoverTaskInstanceList(String cmdParam) {
         Map<String, String> paramMap = JSONUtils.toMap(cmdParam);
 
+        // todo: Can we use a better way to set the recover taskInstanceId list? rather then use the cmdParam
         if (paramMap != null && paramMap.containsKey(CMD_PARAM_RECOVERY_START_NODE_STRING)) {
-            String[] idList = paramMap.get(CMD_PARAM_RECOVERY_START_NODE_STRING).split(COMMA);
-            instanceList = getRecoverTaskInstanceList(idList);
+            String[] idList = paramMap.get(CMD_PARAM_RECOVERY_START_NODE_STRING).split(Constants.COMMA);
+            if (ArrayUtils.isNotEmpty(idList)) {
+                List<Integer> taskInstanceIds = Arrays.stream(idList)
+                    .map(Integer::valueOf)
+                    .collect(Collectors.toList());
+                return processService.findTaskInstanceByIdList(taskInstanceIds);
+            }
         }
-        return instanceList;
+        return Collections.emptyList();
     }
 
     /**
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowSubmitStatue.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowSubmitStatue.java
new file mode 100644
index 0000000000..b53a500c89
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowSubmitStatue.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.runner;
+
+public enum WorkflowSubmitStatue {
+    /**
+     * Submit success
+     */
+    SUCCESS,
+    /**
+     * Submit failed, this status should be retry
+     */
+    FAILED,
+    /**
+     * Duplicated submitted, this status should never occur.
+     */
+    DUPLICATED_SUBMITTED,
+    ;
+}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
similarity index 93%
rename from dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
rename to dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
index f2d263f699..bc3051ae14 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
@@ -15,12 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.server.master;
+package org.apache.dolphinscheduler.server.master.runner;
 
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
+
 import static org.powermock.api.mockito.PowerMockito.mock;
 
 import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -35,9 +36,8 @@ import org.apache.dolphinscheduler.dao.entity.Schedule;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.runner.StateWheelExecuteThread;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
-import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
+import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
+import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
@@ -70,7 +70,7 @@ import org.springframework.context.ApplicationContext;
  */
 @RunWith(PowerMockRunner.class)
 @PrepareForTest({WorkflowExecuteRunnable.class})
-public class WorkflowExecuteTaskTest {
+public class WorkflowExecuteRunnableTest {
 
     private WorkflowExecuteRunnable workflowExecuteThread;
 
@@ -113,7 +113,10 @@ public class WorkflowExecuteTaskTest {
         Mockito.when(processInstance.getProcessDefinition()).thenReturn(processDefinition);
 
         stateWheelExecuteThread = mock(StateWheelExecuteThread.class);
-        workflowExecuteThread = PowerMockito.spy(new WorkflowExecuteRunnable(processInstance, processService, null, null, config, stateWheelExecuteThread));
+        NettyExecutorManager nettyExecutorManager = mock(NettyExecutorManager.class);
+        ProcessAlertManager processAlertManager = mock(ProcessAlertManager.class);
+        workflowExecuteThread =
+            PowerMockito.spy(new WorkflowExecuteRunnable(processInstance, processService, nettyExecutorManager, processAlertManager, config, stateWheelExecuteThread));
         // prepareProcess init dag
         Field dag = WorkflowExecuteRunnable.class.getDeclaredField("dag");
         dag.setAccessible(true);
@@ -154,9 +157,9 @@ public class WorkflowExecuteTaskTest {
                     Arrays.asList(taskInstance1.getId(), taskInstance2.getId(), taskInstance3.getId(), taskInstance4.getId()))
             ).thenReturn(Arrays.asList(taskInstance1, taskInstance2, taskInstance3, taskInstance4));
             Class<WorkflowExecuteRunnable> masterExecThreadClass = WorkflowExecuteRunnable.class;
-            Method method = masterExecThreadClass.getDeclaredMethod("getStartTaskInstanceList", String.class);
+            Method method = masterExecThreadClass.getDeclaredMethod("getRecoverTaskInstanceList", String.class);
             method.setAccessible(true);
-            List<TaskInstance> taskInstances = (List<TaskInstance>) method.invoke(workflowExecuteThread, JSONUtils.toJsonString(cmdParam));
+            List<TaskInstance> taskInstances = workflowExecuteThread.getRecoverTaskInstanceList(JSONUtils.toJsonString(cmdParam));
             Assert.assertEquals(4, taskInstances.size());
 
             cmdParam.put(CMD_PARAM_RECOVERY_START_NODE_STRING, "1");
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index 588fb59422..4bfb0b4809 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -81,7 +81,7 @@ public interface ProcessService {
 
     ProcessDefinition findProcessDefineById(int processDefinitionId);
 
-    ProcessDefinition findProcessDefinition(Long processDefinitionCode, int version);
+    ProcessDefinition findProcessDefinition(Long processDefinitionCode, int processDefinitionVersion);
 
     ProcessDefinition findProcessDefinitionByCode(Long processDefinitionCode);
 


[dolphinscheduler] 06/29: Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe (#10479)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 9a4c7f876aafb54cdd28a889ffb1b7856022c448
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Thu Jun 16 21:46:18 2022 +0800

    Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe (#10479)
    
    * Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe
    
    (cherry picked from commit ad2646ff1f7baa5d76d29023ced2c28a89b52f6b)
---
 .../server/master/runner/WorkflowExecuteRunnable.java                | 5 +++++
 .../server/master/runner/WorkflowExecuteThreadPool.java              | 2 +-
 .../server/master/runner/task/CommonTaskProcessor.java               | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 014abd30bc..2b11498a32 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -2050,6 +2050,11 @@ public class WorkflowExecuteRunnable implements Runnable {
     }
 
     private void measureTaskState(StateEvent taskStateEvent) {
+        if (taskStateEvent == null || taskStateEvent.getExecutionStatus() == null) {
+            // the event is broken
+            logger.warn("The task event is broken..., taskEvent: {}", taskStateEvent);
+            return;
+        }
         if (taskStateEvent.getExecutionStatus().typeIsFinished()) {
             TaskMetrics.incTaskFinish();
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index d8e2fbbeda..92ff7c04f1 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -107,7 +107,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     /**
      * Handle the events belong to the given workflow.
      */
-    public void executeEvent(WorkflowExecuteRunnable workflowExecuteThread) {
+    public void executeEvent(final WorkflowExecuteRunnable workflowExecuteThread) {
         if (!workflowExecuteThread.isStart() || workflowExecuteThread.eventSize() == 0) {
             return;
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
index ae8198ebfd..a19d08affc 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
@@ -125,7 +125,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
             taskPriority.setTaskExecutionContext(taskExecutionContext);
 
             taskUpdateQueue.put(taskPriority);
-            logger.info("master submit success, task : {}", taskInstance.getName());
+            logger.info("Master submit task to priority queue success, taskInstanceId : {}", taskInstance.getId());
             return true;
         } catch (Exception e) {
             logger.error("submit task error", e);


[dolphinscheduler] 29/29: Return status

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 6df8593add92ae8f6cbfcc380a693f0dbaf4e93b
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jul 19 12:24:59 2022 +0800

    Return status
---
 .../dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 0e94b0318d..10c54a5b3a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -1524,6 +1524,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         if (state == ExecutionStatus.READY_BLOCK) {
             ExecutionStatus executionStatus = processReadyBlock();
             logger.info("The workflowInstance is ready to block, the workflowInstance status is {}", executionStatus);
+            return executionStatus
         }
 
         // waiting thread
@@ -1552,6 +1553,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
                 executionStatus = ExecutionStatus.SUCCESS;
             }
             logger.info("The workflowInstance is ready to stop, the workflow status is {}", executionStatus);
+            return executionStatus;
         }
 
         // process failure


[dolphinscheduler] 01/29: Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe (#10479)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 4ceb42087310575ffedd7b09fbf35a08614bb3b7
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Thu Jun 16 21:46:18 2022 +0800

    Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe (#10479)
    
    * Fix TaskProcessorFactory#getTaskProcessor get common processor is not thread safe
    
    (cherry picked from commit ad2646ff1f7baa5d76d29023ced2c28a89b52f6b)
---
 .../dolphinscheduler/common/utils/FileUtils.java   |  29 +++-
 .../dao/entity/ProcessInstance.java                |  12 +-
 .../master/consumer/TaskPriorityQueueConsumer.java |   8 +-
 .../server/master/processor/queue/TaskEvent.java   | 105 +-----------
 .../processor/queue/TaskExecuteThreadPool.java     |  22 +--
 .../master/runner/WorkflowExecuteThread.java       | 176 +++++++++++++--------
 .../master/runner/WorkflowExecuteThreadPool.java   |   2 +-
 .../master/runner/task/CommonTaskProcessor.java    |   4 +-
 .../master/runner/task/TaskProcessorFactory.java   |  27 +++-
 .../runner/task/TaskProcessorFactoryTest.java      |   4 +-
 .../remote/utils/ChannelUtils.java                 |  12 +-
 .../apache/dolphinscheduler/remote/utils/Host.java |   2 +
 .../service/process/ProcessServiceImpl.java        |   5 +-
 .../queue/PeerTaskInstancePriorityQueue.java       |  35 ++--
 .../queue/PeerTaskInstancePriorityQueueTest.java   |   9 +-
 .../src/main/resources/application.yaml            |   4 +-
 .../plugin/task/shell/ShellTask.java               |   7 +-
 .../server/worker/config/WorkerConfig.java         |   4 +-
 .../worker/processor/TaskCallbackService.java      |   2 +
 .../server/worker/runner/TaskExecuteThread.java    |  24 ++-
 20 files changed, 247 insertions(+), 246 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index bdcf62f76a..23e4b74b75 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -17,14 +17,25 @@
 
 package org.apache.dolphinscheduler.common.utils;
 
+import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
+import static org.apache.dolphinscheduler.common.Constants.FOLDER_SEPARATOR;
+import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXES;
+import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE;
+import static org.apache.dolphinscheduler.common.Constants.UTF_8;
+import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
+
 import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.NoSuchFileException;
 
-import static org.apache.dolphinscheduler.common.Constants.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * file utils
@@ -112,7 +123,15 @@ public class FileUtils {
         File execLocalPathFile = new File(execLocalPath);
 
         if (execLocalPathFile.exists()) {
-            org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
+            try {
+                org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
+            } catch (Exception ex) {
+                if (ex instanceof NoSuchFileException || ex.getCause() instanceof NoSuchFileException) {
+                    // this file is already be deleted.
+                } else {
+                    throw ex;
+                }
+            }
         }
 
         //create work dir
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
index 8f639efc07..fb3ae13858 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessInstance.java
@@ -266,13 +266,11 @@ public class ProcessInstance {
      */
     public ProcessInstance(ProcessDefinition processDefinition) {
         this.processDefinition = processDefinition;
-        this.name = processDefinition.getName()
-            + "-"
-            +
-            processDefinition.getVersion()
-            + "-"
-            +
-            DateUtils.getCurrentTimeStamp();
+        // todo: the name is not unique
+        this.name = String.join("-",
+                processDefinition.getName(),
+                String.valueOf(processDefinition.getVersion()),
+                DateUtils.getCurrentTimeStamp());
     }
 
     public String getVarPool() {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index f692685009..f9c19c48fd 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -180,16 +180,18 @@ public class TaskPriorityQueueConsumer extends Thread {
                     return true;
                 }
             }
-
             result = dispatcher.dispatch(executionContext);
 
             if (result) {
+                logger.info("Master success dispatch task to worker, taskInstanceId: {}", taskPriority.getTaskId());
                 addDispatchEvent(context, executionContext);
+            } else {
+                logger.info("Master failed to dispatch task to worker, taskInstanceId: {}", taskPriority.getTaskId());
             }
         } catch (RuntimeException e) {
-            logger.error("dispatch error: ", e);
+            logger.error("Master dispatch task to worker error: ", e);
         } catch (ExecuteException e) {
-            logger.error("dispatch error: {}", e.getMessage());
+            logger.error("Master dispatch task to worker error: {}", e);
         }
         return result;
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
index 865eee53a5..86cce5170e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
@@ -26,10 +26,12 @@ import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
 import java.util.Date;
 
 import io.netty.channel.Channel;
+import lombok.Data;
 
 /**
  * task event
  */
+@Data
 public class TaskEvent {
 
     /**
@@ -135,107 +137,4 @@ public class TaskEvent {
         return event;
     }
 
-    public String getVarPool() {
-        return varPool;
-    }
-
-    public void setVarPool(String varPool) {
-        this.varPool = varPool;
-    }
-
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-    }
-
-    public String getWorkerAddress() {
-        return workerAddress;
-    }
-
-    public void setWorkerAddress(String workerAddress) {
-        this.workerAddress = workerAddress;
-    }
-
-    public ExecutionStatus getState() {
-        return state;
-    }
-
-    public void setState(ExecutionStatus state) {
-        this.state = state;
-    }
-
-    public Date getStartTime() {
-        return startTime;
-    }
-
-    public void setStartTime(Date startTime) {
-        this.startTime = startTime;
-    }
-
-    public Date getEndTime() {
-        return endTime;
-    }
-
-    public void setEndTime(Date endTime) {
-        this.endTime = endTime;
-    }
-
-    public String getExecutePath() {
-        return executePath;
-    }
-
-    public void setExecutePath(String executePath) {
-        this.executePath = executePath;
-    }
-
-    public String getLogPath() {
-        return logPath;
-    }
-
-    public void setLogPath(String logPath) {
-        this.logPath = logPath;
-    }
-
-    public int getProcessId() {
-        return processId;
-    }
-
-    public void setProcessId(int processId) {
-        this.processId = processId;
-    }
-
-    public String getAppIds() {
-        return appIds;
-    }
-
-    public void setAppIds(String appIds) {
-        this.appIds = appIds;
-    }
-
-    public Event getEvent() {
-        return event;
-    }
-
-    public void setEvent(Event event) {
-        this.event = event;
-    }
-
-    public Channel getChannel() {
-        return channel;
-    }
-
-    public void setChannel(Channel channel) {
-        this.channel = channel;
-    }
-
-    public int getProcessInstanceId() {
-        return processInstanceId;
-    }
-
-    public void setProcessInstanceId(int processInstanceId) {
-        this.processInstanceId = processInstanceId;
-    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
index 75c0272519..c9c2868d36 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
@@ -79,7 +79,7 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
 
     public void submitTaskEvent(TaskEvent taskEvent) {
         if (!processInstanceExecCacheManager.contains(taskEvent.getProcessInstanceId())) {
-            logger.warn("workflowExecuteThread is null, event: {}", taskEvent);
+            logger.warn("Cannot find workflowExecuteThread from cacheManager, event: {}", taskEvent);
             return;
         }
         if (!taskExecuteThreadMap.containsKey(taskEvent.getProcessInstanceId())) {
@@ -114,20 +114,24 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
         future.addCallback(new ListenableFutureCallback() {
             @Override
             public void onFailure(Throwable ex) {
-                logger.error("handle event {} failed: {}", taskExecuteThread.getProcessInstanceId(), ex);
-                if (!processInstanceExecCacheManager.contains(taskExecuteThread.getProcessInstanceId())) {
-                    taskExecuteThreadMap.remove(taskExecuteThread.getProcessInstanceId());
-                    logger.info("remove process instance: {}", taskExecuteThread.getProcessInstanceId());
+                Integer processInstanceId = taskExecuteThread.getProcessInstanceId();
+                logger.error("persist event failed processInstanceId: {}", processInstanceId, ex);
+                if (!processInstanceExecCacheManager.contains(processInstanceId)) {
+                    taskExecuteThreadMap.remove(processInstanceId);
+                    logger.info("Cannot find processInstance from cacheManager, remove process instance from threadMap: {}",
+                            processInstanceId);
                 }
                 multiThreadFilterMap.remove(taskExecuteThread.getKey());
             }
 
             @Override
             public void onSuccess(Object result) {
-                logger.info("persist events {} succeeded.", taskExecuteThread.getProcessInstanceId());
-                if (!processInstanceExecCacheManager.contains(taskExecuteThread.getProcessInstanceId())) {
-                    taskExecuteThreadMap.remove(taskExecuteThread.getProcessInstanceId());
-                    logger.info("remove process instance: {}", taskExecuteThread.getProcessInstanceId());
+                Integer processInstanceId = taskExecuteThread.getProcessInstanceId();
+                logger.info("persist events succeeded, processInstanceId: {}", processInstanceId);
+                if (!processInstanceExecCacheManager.contains(processInstanceId)) {
+                    taskExecuteThreadMap.remove(processInstanceId);
+                    logger.info("Cannot find processInstance from cacheManager, remove process instance from threadMap: {}",
+                            processInstanceId);
                 }
                 multiThreadFilterMap.remove(taskExecuteThread.getKey());
             }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
index 993e1df14b..fede5abad2 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
@@ -75,8 +75,8 @@ import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
 import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
 
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -87,6 +87,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -238,12 +239,12 @@ public class WorkflowExecuteThread {
      * @param masterConfig            masterConfig
      * @param stateWheelExecuteThread stateWheelExecuteThread
      */
-    public WorkflowExecuteThread(ProcessInstance processInstance
-        , ProcessService processService
-        , NettyExecutorManager nettyExecutorManager
-        , ProcessAlertManager processAlertManager
-        , MasterConfig masterConfig
-        , StateWheelExecuteThread stateWheelExecuteThread) {
+    public WorkflowExecuteThread(ProcessInstance processInstance,
+                                 ProcessService processService,
+                                 NettyExecutorManager nettyExecutorManager,
+                                 ProcessAlertManager processAlertManager,
+                                 MasterConfig masterConfig,
+                                 StateWheelExecuteThread stateWheelExecuteThread) {
         this.processService = processService;
         this.processInstance = processInstance;
         this.masterConfig = masterConfig;
@@ -279,15 +280,14 @@ public class WorkflowExecuteThread {
     }
 
     public String getKey() {
-        if (StringUtils.isNotEmpty(key)
-            || this.processDefinition == null) {
+        if (StringUtils.isNotEmpty(key) || this.processDefinition == null) {
             return key;
         }
 
         key = String.format("%d_%d_%d",
-            this.processDefinition.getCode(),
-            this.processDefinition.getVersion(),
-            this.processInstance.getId());
+                            this.processDefinition.getCode(),
+                            this.processDefinition.getVersion(),
+                            this.processInstance.getId());
         return key;
     }
 
@@ -436,10 +436,10 @@ public class WorkflowExecuteThread {
 
     private void taskFinished(TaskInstance taskInstance) {
         logger.info("work flow {} task id:{} code:{} state:{} ",
-            processInstance.getId(),
-            taskInstance.getId(),
-            taskInstance.getTaskCode(),
-            taskInstance.getState());
+                    processInstance.getId(),
+                    taskInstance.getId(),
+                    taskInstance.getTaskCode(),
+                    taskInstance.getState());
 
         activeTaskProcessorMaps.remove(taskInstance.getTaskCode());
         stateWheelExecuteThread.removeTask4TimeoutCheck(processInstance, taskInstance);
@@ -460,7 +460,7 @@ public class WorkflowExecuteThread {
             completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
             // There are child nodes and the failure policy is: CONTINUE
             if (DagHelper.haveAllNodeAfterNode(Long.toString(taskInstance.getTaskCode()), dag)
-                    && processInstance.getFailureStrategy() == FailureStrategy.CONTINUE) {
+                && processInstance.getFailureStrategy() == FailureStrategy.CONTINUE) {
                 submitPostNode(Long.toString(taskInstance.getTaskCode()));
             } else {
                 errorTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
@@ -492,8 +492,9 @@ public class WorkflowExecuteThread {
                     this.stateEvents.add(nextEvent);
                 } else {
                     ProcessInstance processInstance = this.processService.findProcessInstanceById(nextTaskInstance.getProcessInstanceId());
-                    this.processService.sendStartTask2Master(processInstance, nextTaskInstance.getId(),
-                        org.apache.dolphinscheduler.remote.command.CommandType.TASK_WAKEUP_EVENT_REQUEST);
+                    this.processService.sendStartTask2Master(processInstance,
+                                                             nextTaskInstance.getId(),
+                                                             org.apache.dolphinscheduler.remote.command.CommandType.TASK_WAKEUP_EVENT_REQUEST);
                 }
             }
         }
@@ -515,7 +516,8 @@ public class WorkflowExecuteThread {
         }
         waitToRetryTaskInstanceMap.put(newTaskInstance.getTaskCode(), newTaskInstance);
         if (!taskInstance.retryTaskIntervalOverTime()) {
-            logger.info("failure task will be submitted: process id: {}, task instance code: {} state:{} retry times:{} / {}, interval:{}",
+            logger.info(
+                "failure task will be submitted: process id: {}, task instance code: {} state:{} retry times:{} / {}, interval:{}",
                 processInstance.getId(),
                 newTaskInstance.getTaskCode(),
                 newTaskInstance.getState(),
@@ -552,7 +554,7 @@ public class WorkflowExecuteThread {
         logger.info("process instance update: {}", processInstanceId);
         processInstance = processService.findProcessInstanceById(processInstanceId);
         processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(),
-            processInstance.getProcessDefinitionVersion());
+                                                                 processInstance.getProcessDefinitionVersion());
         processInstance.setProcessDefinition(processDefinition);
     }
 
@@ -580,9 +582,7 @@ public class WorkflowExecuteThread {
      */
     public boolean checkProcessInstance(StateEvent stateEvent) {
         if (this.processInstance.getId() != stateEvent.getProcessInstanceId()) {
-            logger.error("mismatch process instance id: {}, state event:{}",
-                this.processInstance.getId(),
-                stateEvent);
+            logger.error("mismatch process instance id: {}, state event:{}", this.processInstance.getId(), stateEvent);
             return false;
         }
         return true;
@@ -742,9 +742,9 @@ public class WorkflowExecuteThread {
                 return true;
             }
             logger.info("process complement continue. process id:{}, schedule time:{} complementListDate:{}",
-                processInstance.getId(),
-                processInstance.getScheduleTime(),
-                complementListDate.toString());
+                        processInstance.getId(),
+                        processInstance.getScheduleTime(),
+                        complementListDate.toString());
             scheduleDate = complementListDate.get(index + 1);
         }
         //the next process complement
@@ -783,8 +783,7 @@ public class WorkflowExecuteThread {
     }
 
     private boolean needComplementProcess() {
-        if (processInstance.isComplementData()
-            && Flag.NO == processInstance.getIsSubProcess()) {
+        if (processInstance.isComplementData() && Flag.NO == processInstance.getIsSubProcess()) {
             return true;
         }
         return false;
@@ -863,7 +862,7 @@ public class WorkflowExecuteThread {
             return;
         }
         processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(),
-            processInstance.getProcessDefinitionVersion());
+                                                                 processInstance.getProcessDefinitionVersion());
         processInstance.setProcessDefinition(processDefinition);
 
         List<TaskInstance> recoverNodeList = getStartTaskInstanceList(processInstance.getCommandParam());
@@ -883,7 +882,9 @@ public class WorkflowExecuteThread {
         List<String> recoveryNodeCodeList = getRecoveryNodeCodeList(recoverNodeList);
         List<String> startNodeNameList = parseStartNodeName(processInstance.getCommandParam());
         ProcessDag processDag = generateFlowDag(taskNodeList,
-            startNodeNameList, recoveryNodeCodeList, processInstance.getTaskDependType());
+                                                startNodeNameList,
+                                                recoveryNodeCodeList,
+                                                processInstance.getTaskDependType());
         if (processDag == null) {
             logger.error("processDag is null");
             return;
@@ -955,14 +956,16 @@ public class WorkflowExecuteThread {
                 if (complementListDate.size() == 0 && needComplementProcess()) {
                     complementListDate = CronUtils.getSelfFireDateList(start, end, schedules);
                     logger.info(" process definition code:{} complement data: {}",
-                        processInstance.getProcessDefinitionCode(), complementListDate.toString());
+                                processInstance.getProcessDefinitionCode(),
+                                complementListDate.toString());
 
                     if (complementListDate.size() > 0 && Flag.NO == processInstance.getIsSubProcess()) {
                         processInstance.setScheduleTime(complementListDate.get(0));
-                        processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
-                            processDefinition.getGlobalParamMap(),
-                            processDefinition.getGlobalParamList(),
-                            CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime(), cmdParam.get(Constants.SCHEDULE_TIMEZONE)));
+                        processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(processDefinition.getGlobalParamMap(),
+                                                                                          processDefinition.getGlobalParamList(),
+                                                                                          CommandType.COMPLEMENT_DATA,
+                                                                                          processInstance.getScheduleTime(),
+                                                                                          cmdParam.get(Constants.SCHEDULE_TIMEZONE)));
                         processService.updateProcessInstance(processInstance);
                     }
                 }
@@ -976,7 +979,7 @@ public class WorkflowExecuteThread {
      * @param taskInstance task instance
      * @return TaskInstance
      */
-    private TaskInstance submitTaskExec(TaskInstance taskInstance) {
+    private Optional<TaskInstance> submitTaskExec(TaskInstance taskInstance) {
         try {
             // package task instance before submit
             processService.packageTaskInstance(taskInstance, processInstance);
@@ -984,17 +987,17 @@ public class WorkflowExecuteThread {
             ITaskProcessor taskProcessor = TaskProcessorFactory.getTaskProcessor(taskInstance.getTaskType());
             taskProcessor.init(taskInstance, processInstance);
 
-            if (taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION
-                && taskProcessor.getType().equalsIgnoreCase(Constants.COMMON_TASK_TYPE)) {
+            if (taskInstance.getState() == ExecutionStatus.RUNNING_EXECUTION && taskProcessor.getType()
+                .equalsIgnoreCase(Constants.COMMON_TASK_TYPE)) {
                 notifyProcessHostUpdate(taskInstance);
             }
 
             boolean submit = taskProcessor.action(TaskAction.SUBMIT);
             if (!submit) {
                 logger.error("process id:{} name:{} submit standby task id:{} name:{} failed!",
-                    processInstance.getId(), processInstance.getName(),
-                    taskInstance.getId(), taskInstance.getName());
-                return null;
+                        processInstance.getId(), processInstance.getName(),
+                        taskInstance.getId(), taskInstance.getName());
+                return Optional.empty();
             }
 
             // in a dag, only one taskInstance is valid per taskCode, so need to set the old taskInstance invalid
@@ -1033,10 +1036,10 @@ public class WorkflowExecuteThread {
                 taskStateChangeEvent.setType(StateEventType.TASK_STATE_CHANGE);
                 this.stateEvents.add(taskStateChangeEvent);
             }
-            return taskInstance;
+            return Optional.of(taskInstance);
         } catch (Exception e) {
             logger.error("submit standby task error", e);
-            return null;
+            return Optional.empty();
         }
     }
 
@@ -1333,6 +1336,7 @@ public class WorkflowExecuteThread {
         for (TaskInstance task : taskInstances) {
 
             if (readyToSubmitTaskQueue.contains(task)) {
+                logger.warn("Task is already at submit queue, taskInstanceId: {}", task.getId());
                 continue;
             }
 
@@ -1659,11 +1663,12 @@ public class WorkflowExecuteThread {
     private void updateProcessInstanceState() {
         ExecutionStatus state = getProcessInstanceState(processInstance);
         if (processInstance.getState() != state) {
-            logger.info(
-                "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
-                processInstance.getId(), processInstance.getName(),
-                processInstance.getState(), state,
-                processInstance.getCommandType());
+            logger.info("work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
+                        processInstance.getId(),
+                        processInstance.getName(),
+                        processInstance.getState(),
+                        state,
+                        processInstance.getCommandType());
 
             processInstance.setState(state);
             if (state.typeIsFinished()) {
@@ -1687,11 +1692,12 @@ public class WorkflowExecuteThread {
     private void updateProcessInstanceState(StateEvent stateEvent) {
         ExecutionStatus state = stateEvent.getExecutionStatus();
         if (processInstance.getState() != state) {
-            logger.info(
-                "work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
-                processInstance.getId(), processInstance.getName(),
-                processInstance.getState(), state,
-                processInstance.getCommandType());
+            logger.info("work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
+                        processInstance.getId(),
+                        processInstance.getName(),
+                        processInstance.getState(),
+                        state,
+                        processInstance.getCommandType());
 
             processInstance.setState(state);
             if (state.typeIsFinished()) {
@@ -1723,7 +1729,9 @@ public class WorkflowExecuteThread {
                 return;
             }
             logger.info("add task to stand by list, task name:{}, task id:{}, task code:{}",
-                taskInstance.getName(), taskInstance.getId(), taskInstance.getTaskCode());
+                        taskInstance.getName(),
+                        taskInstance.getId(),
+                        taskInstance.getTaskCode());
             readyToSubmitTaskQueue.put(taskInstance);
         } catch (Exception e) {
             logger.error("add task instance to readyToSubmitTaskQueue, taskName:{}, task id:{}", taskInstance.getName(), taskInstance.getId(), e);
@@ -1736,15 +1744,14 @@ public class WorkflowExecuteThread {
      * @param taskInstance task instance
      */
     private void removeTaskFromStandbyList(TaskInstance taskInstance) {
-        logger.info("remove task from stand by list, id: {} name:{}",
-            taskInstance.getId(),
-            taskInstance.getName());
+        logger.info("remove task from stand by list, id: {} name:{}", taskInstance.getId(), taskInstance.getName());
         try {
             readyToSubmitTaskQueue.remove(taskInstance);
         } catch (Exception e) {
             logger.error("remove task instance from readyToSubmitTaskQueue error, task id:{}, Name: {}",
-                taskInstance.getId(),
-                taskInstance.getName(), e);
+                         taskInstance.getId(),
+                         taskInstance.getName(),
+                         e);
         }
     }
 
@@ -1766,8 +1773,9 @@ public class WorkflowExecuteThread {
      * close the on going tasks
      */
     private void killAllTasks() {
-        logger.info("kill called on process instance id: {}, num: {}", processInstance.getId(),
-            activeTaskProcessorMaps.size());
+        logger.info("kill called on process instance id: {}, num: {}",
+                    processInstance.getId(),
+                    activeTaskProcessorMaps.size());
 
         if (readyToSubmitTaskQueue.size() > 0) {
             readyToSubmitTaskQueue.clear();
@@ -1831,14 +1839,16 @@ public class WorkflowExecuteThread {
                 }
                 DependResult dependResult = getDependResultForTask(task);
                 if (DependResult.SUCCESS == dependResult) {
-                    TaskInstance taskInstance = submitTaskExec(task);
-                    if (taskInstance == null) {
+                    Optional<TaskInstance> taskInstanceOptional = submitTaskExec(task);
+                    if (!taskInstanceOptional.isPresent()) {
                         this.taskFailedSubmit = true;
                         // Remove and add to complete map and error map
                         removeTaskFromStandbyList(task);
                         completeTaskMap.put(task.getTaskCode(), task.getId());
                         errorTaskMap.put(task.getTaskCode(), task.getId());
-                        logger.error("process {}, task {}, code:{} submit task failed.", task.getProcessInstanceId(), task.getName(), task.getTaskCode());
+                        logger.error("Task submitted failed, processInstanceId: {}, taskInstanceId: {}",
+                                     task.getProcessInstanceId(),
+                                     task.getId());
                     } else {
                         removeTaskFromStandbyList(task);
                     }
@@ -1846,11 +1856,15 @@ public class WorkflowExecuteThread {
                     // if the dependency fails, the current node is not submitted and the state changes to failure.
                     dependFailedTaskMap.put(task.getTaskCode(), task.getId());
                     removeTaskFromStandbyList(task);
-                    logger.info("task {},id:{} depend result : {}", task.getName(), task.getId(), dependResult);
+                    logger.info("Task dependent result is failed, taskInstanceId:{} depend result : {}",
+                                task.getId(),
+                                dependResult);
                 } else if (DependResult.NON_EXEC == dependResult) {
                     // for some reasons(depend task pause/stop) this task would not be submit
                     removeTaskFromStandbyList(task);
-                    logger.info("remove task {},id:{} , because depend result : {}", task.getName(), task.getId(), dependResult);
+                    logger.info("Remove task due to depend result not executed, taskInstanceId:{} depend result : {}",
+                                task.getId(),
+                                dependResult);
                 }
             }
         } catch (Exception e) {
@@ -2009,4 +2023,30 @@ public class WorkflowExecuteThread {
             }
         }
     }
+
 }
+
+    private void measureTaskState(StateEvent taskStateEvent) {
+        if (taskStateEvent == null || taskStateEvent.getExecutionStatus() == null) {
+            // the event is broken
+            logger.warn("The task event is broken..., taskEvent: {}", taskStateEvent);
+            return;
+        }
+        if (taskStateEvent.getExecutionStatus().typeIsFinished()) {
+            TaskMetrics.incTaskFinish();
+        }
+        switch (taskStateEvent.getExecutionStatus()) {
+            case STOP:
+                TaskMetrics.incTaskStop();
+                break;
+            case SUCCESS:
+                TaskMetrics.incTaskSuccess();
+                break;
+            case FAILURE:
+                TaskMetrics.incTaskFailure();
+                break;
+            default:
+                break;
+        }
+    }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index cd337b4c95..556f36a1b7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -100,7 +100,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     /**
      * execute workflow
      */
-    public void executeEvent(WorkflowExecuteThread workflowExecuteThread) {
+    public void executeEvent(final WorkflowExecuteRunnable workflowExecuteThread) {
         if (!workflowExecuteThread.isStart() || workflowExecuteThread.eventSize() == 0) {
             return;
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
index ffeb89a0d2..81e3070089 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
@@ -110,7 +110,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
                 logger.info("submit task, but the status of the task {} is already running or delayed.", taskInstance.getName());
                 return true;
             }
-            logger.info("task ready to submit: {}", taskInstance);
+            logger.info("task ready to submit: taskInstanceId: {}", taskInstance.getId());
 
             TaskPriority taskPriority = new TaskPriority(processInstance.getProcessInstancePriority().getCode(),
                     processInstance.getId(), taskInstance.getProcessInstancePriority().getCode(),
@@ -125,7 +125,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
             taskPriority.setTaskExecutionContext(taskExecutionContext);
 
             taskUpdateQueue.put(taskPriority);
-            logger.info(String.format("master submit success, task : %s", taskInstance.getName()));
+            logger.info("Master submit task to priority queue success, taskInstanceId : {}", taskInstance.getId());
             return true;
         } catch (Exception e) {
             logger.error("submit task error", e);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
index 0129338649..d5bd131a10 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
@@ -21,36 +21,47 @@ import static org.apache.dolphinscheduler.common.Constants.COMMON_TASK_TYPE;
 
 import org.apache.commons.lang3.StringUtils;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.Map;
-import java.util.Objects;
 import java.util.ServiceLoader;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * the factory to create task processor
  */
 public class TaskProcessorFactory {
 
-    public static final Map<String, ITaskProcessor> PROCESS_MAP = new ConcurrentHashMap<>();
+    private static final Logger logger = LoggerFactory.getLogger(TaskProcessorFactory.class);
+
+    public static final Map<String, Constructor<ITaskProcessor>> PROCESS_MAP = new ConcurrentHashMap<>();
 
     private static final String DEFAULT_PROCESSOR = COMMON_TASK_TYPE;
 
     static {
         for (ITaskProcessor iTaskProcessor : ServiceLoader.load(ITaskProcessor.class)) {
-            PROCESS_MAP.put(iTaskProcessor.getType(), iTaskProcessor);
+            try {
+                PROCESS_MAP.put(iTaskProcessor.getType(), (Constructor<ITaskProcessor>) iTaskProcessor.getClass().getConstructor());
+            } catch (NoSuchMethodException e) {
+                throw new IllegalArgumentException("The task processor should has a no args constructor");
+            }
         }
     }
 
-    public static ITaskProcessor getTaskProcessor(String type) throws InstantiationException, IllegalAccessException {
+    public static ITaskProcessor getTaskProcessor(String type) throws InvocationTargetException, InstantiationException, IllegalAccessException {
         if (StringUtils.isEmpty(type)) {
             type = DEFAULT_PROCESSOR;
         }
-        ITaskProcessor iTaskProcessor = PROCESS_MAP.get(type);
-        if (Objects.isNull(iTaskProcessor)) {
-            iTaskProcessor = PROCESS_MAP.get(DEFAULT_PROCESSOR);
+        Constructor<ITaskProcessor> iTaskProcessorConstructor = PROCESS_MAP.get(type);
+        if (iTaskProcessorConstructor == null) {
+            logger.warn("ITaskProcessor could not found for taskType: {}", type);
+            iTaskProcessorConstructor = PROCESS_MAP.get(DEFAULT_PROCESSOR);
         }
 
-        return iTaskProcessor.getClass().newInstance();
+        return iTaskProcessorConstructor.newInstance();
     }
 
     /**
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
index d0371809cc..b974a40b31 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactoryTest.java
@@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.server.master.runner.task;
 
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 
+import java.lang.reflect.InvocationTargetException;
+
 import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -27,7 +29,7 @@ import org.junit.Test;
 public class TaskProcessorFactoryTest {
 
     @Test
-    public void testFactory() throws InstantiationException, IllegalAccessException {
+    public void testFactory() throws InvocationTargetException, InstantiationException, IllegalAccessException {
 
         TaskInstance taskInstance = new TaskInstance();
         taskInstance.setTaskType("shell");
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java
index 239a3993c0..b4177ec25d 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java
@@ -21,6 +21,9 @@ import org.apache.dolphinscheduler.common.utils.NetUtils;
 
 import java.net.InetSocketAddress;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import io.netty.channel.Channel;
 
 /**
@@ -28,6 +31,8 @@ import io.netty.channel.Channel;
  */
 public class ChannelUtils {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(ChannelUtils.class);
+
     private ChannelUtils() {
         throw new IllegalStateException(ChannelUtils.class.getName());
     }
@@ -49,7 +54,7 @@ public class ChannelUtils {
      * @return remote address
      */
     public static String getRemoteAddress(Channel channel) {
-        return NetUtils.getHost(((InetSocketAddress) channel.remoteAddress()).getAddress());
+        return toAddress(channel).getAddress();
     }
 
     /**
@@ -60,6 +65,11 @@ public class ChannelUtils {
      */
     public static Host toAddress(Channel channel) {
         InetSocketAddress socketAddress = ((InetSocketAddress) channel.remoteAddress());
+        if (socketAddress == null) {
+            // the remote channel already closed
+            LOGGER.warn("The channel is already closed");
+            return Host.EMPTY;
+        }
         return new Host(NetUtils.getHost(socketAddress.getAddress()), socketAddress.getPort());
     }
 
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java
index 2163e9c7d8..dc8e1f0d36 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java
@@ -27,6 +27,8 @@ import java.util.Objects;
  */
 public class Host implements Serializable {
 
+    public static final Host EMPTY = new Host();
+
     /**
      * address
      */
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 48bb9c624d..eba7ced00a 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -748,7 +748,8 @@ public class ProcessServiceImpl implements ProcessService {
         processInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
         processInstance.setRecovery(Flag.NO);
         processInstance.setStartTime(new Date());
-        processInstance.setRestartTime(processInstance.getStartTime());
+        // the new process instance restart time is null.
+        processInstance.setRestartTime(null);
         processInstance.setRunTimes(1);
         processInstance.setMaxTryTimes(0);
         processInstance.setCommandParam(command.getCommandParam());
@@ -1266,7 +1267,7 @@ public class ProcessServiceImpl implements ProcessService {
     @Override
     @Transactional(rollbackFor = Exception.class)
     public TaskInstance submitTask(ProcessInstance processInstance, TaskInstance taskInstance) {
-        logger.info("start submit task : {}, instance id:{}, state: {}",
+        logger.info("start submit task : {}, processInstance id:{}, state: {}",
             taskInstance.getName(), taskInstance.getProcessInstanceId(), processInstance.getState());
         //submit to db
         TaskInstance task = submitTaskInstanceToDB(taskInstance, processInstance);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
index 231fd2a20f..2e939ee332 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
@@ -21,11 +21,13 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.service.exceptions.TaskPriorityQueueException;
 
+import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.PriorityQueue;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
 
 /**
  * Task instances priority queue implementation
@@ -40,12 +42,8 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
     /**
      * queue
      */
-    private PriorityQueue<TaskInstance> queue = new PriorityQueue<>(QUEUE_MAX_SIZE, new TaskInfoComparator());
-
-    /**
-     * Lock used for all public operations
-     */
-    private final ReentrantLock lock = new ReentrantLock(true);
+    private final PriorityQueue<TaskInstance> queue = new PriorityQueue<>(QUEUE_MAX_SIZE, new TaskInfoComparator());
+    private final Set<Integer> taskInstanceIdSet = Collections.synchronizedSet(new HashSet<>());
 
     /**
      * put task instance to priority queue
@@ -56,6 +54,7 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
     @Override
     public void put(TaskInstance taskInstance) throws TaskPriorityQueueException {
         queue.add(taskInstance);
+        taskInstanceIdSet.add(taskInstance.getId());
     }
 
     /**
@@ -66,7 +65,11 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      */
     @Override
     public TaskInstance take() throws TaskPriorityQueueException {
-        return queue.poll();
+        TaskInstance taskInstance = queue.poll();
+        if (taskInstance != null) {
+            taskInstanceIdSet.remove(taskInstance.getId());
+        }
+        return taskInstance;
     }
 
     /**
@@ -111,6 +114,7 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      */
     public void clear() {
         queue.clear();
+        taskInstanceIdSet.clear();
     }
 
     /**
@@ -120,20 +124,11 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      * @return true is contains
      */
     public boolean contains(TaskInstance taskInstance) {
-        return this.contains(taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion());
+        return this.contains(taskInstance.getId());
     }
 
-    public boolean contains(long taskCode, int taskVersion) {
-        Iterator<TaskInstance> iterator = this.queue.iterator();
-        while (iterator.hasNext()) {
-            TaskInstance taskInstance = iterator.next();
-            if (taskCode == taskInstance.getTaskCode()
-                    && taskVersion == taskInstance.getTaskDefinitionVersion()) {
-                return true;
-            }
-        }
-        return false;
-
+    public boolean contains(int taskInstanceId) {
+        return taskInstanceIdSet.contains(taskInstanceId);
     }
 
     /**
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java
index 8da3a6c194..67e40d1189 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java
@@ -46,14 +46,11 @@ public class PeerTaskInstancePriorityQueueTest {
         Assert.assertTrue(queue.size() < peekBeforeLength);
     }
 
-    @Test
+
+    @Test(expected = TaskPriorityQueueException.class)
     public void poll() throws Exception {
         PeerTaskInstancePriorityQueue queue = getPeerTaskInstancePriorityQueue();
-        try {
-            queue.poll(1000, TimeUnit.MILLISECONDS);
-        } catch (TaskPriorityQueueException e) {
-            e.printStackTrace();
-        }
+        queue.poll(1000, TimeUnit.MILLISECONDS);
     }
 
     @Test
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index c50883da62..7be661de4e 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -109,7 +109,7 @@ master:
   # master prepare execute thread number to limit handle commands in parallel
   pre-exec-threads: 10
   # master execute thread number to limit process instances in parallel
-  exec-threads: 100
+  exec-threads: 10
   # master dispatch task number per batch
   dispatch-task-number: 3
   # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
@@ -134,7 +134,7 @@ worker:
   # worker listener port
   listen-port: 1234
   # worker execute thread number to limit task instances in parallel
-  exec-threads: 100
+  exec-threads: 10
   # worker heartbeat interval, the unit is second
   heartbeat-interval: 10
   # worker host weight to dispatch tasks, default value 100
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
index 737b876b74..9d084d0ec8 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/src/main/java/org/apache/dolphinscheduler/plugin/task/shell/ShellTask.java
@@ -34,6 +34,7 @@ import org.apache.dolphinscheduler.spi.utils.JSONUtils;
 import org.apache.commons.collections4.MapUtils;
 
 import java.io.File;
+import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
@@ -147,7 +148,11 @@ public class ShellTask extends AbstractTaskExecutor {
             if (!file.getParentFile().exists()) {
                 file.getParentFile().mkdirs();
             }
-            Files.createFile(path, attr);
+            try {
+                Files.createFile(path, attr);
+            } catch (FileAlreadyExistsException ex) {
+                // this is expected
+            }
         }
 
         Files.write(path, shellParameters.getRawScript().getBytes(), StandardOpenOption.APPEND);
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
index 5192e27d61..6edff09f97 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
@@ -21,9 +21,9 @@ import java.util.Set;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.stereotype.Component;
+import org.springframework.context.annotation.Configuration;
 
-@Component
+@Configuration
 @EnableConfigurationProperties
 @ConfigurationProperties("worker")
 public class WorkerConfig {
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
index 3641de8453..2a92222b0a 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
@@ -162,6 +162,8 @@ public class TaskCallbackService {
                     }
                 }
             });
+        } else {
+            logger.warn("Remote channel of taskInstanceId is null: {}, cannot send command: {}", taskInstanceId, command);
         }
     }
 
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
index cc15eb62f4..e6ff2b5653 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@@ -19,12 +19,15 @@ package org.apache.dolphinscheduler.server.worker.runner;
 
 import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH;
 
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.WarningType;
 import org.apache.dolphinscheduler.common.exception.StorageOperateNoConfiguredException;
 import org.apache.dolphinscheduler.common.storage.StorageOperate;
-import org.apache.dolphinscheduler.common.utils.*;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
 import org.apache.dolphinscheduler.plugin.task.api.AbstractTask;
 import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -40,10 +43,17 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager;
 
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.*;
+import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.Delayed;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -233,7 +243,11 @@ public class TaskExecuteThread implements Runnable, Delayed {
                 org.apache.commons.io.FileUtils.deleteDirectory(new File(execLocalPath));
                 logger.info("exec local path: {} cleared.", execLocalPath);
             } catch (IOException e) {
-                logger.error("delete exec dir failed : {}", e.getMessage(), e);
+                if (e instanceof NoSuchFileException) {
+                    // this is expected
+                } else {
+                    logger.error("Delete exec dir failed.", e);
+                }
             }
         }
     }
@@ -264,7 +278,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
                 task.cancelApplication(true);
                 ProcessUtils.killYarnJob(taskExecutionContext);
             } catch (Exception e) {
-                logger.error(e.getMessage(), e);
+                logger.error("Kill task failed", e);
             }
         }
     }


[dolphinscheduler] 07/29: Fix PeerTaskInstancePriorityQueue cannot contains method use taskInstanceId to check (#10491)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 9a590546552e198394e98865c143d639d59ea253
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Fri Jun 17 16:14:56 2022 +0800

    Fix PeerTaskInstancePriorityQueue cannot contains method use taskInstanceId to check (#10491)
    
    (cherry picked from commit 0bdfa0cff958b3ee089d732590e735404bdc81dc)
---
 .../master/runner/WorkflowExecuteRunnable.java     |  2 +-
 .../queue/PeerTaskInstancePriorityQueue.java       | 32 +++++++++++++++-------
 .../queue/PeerTaskInstancePriorityQueueTest.java   |  6 ++++
 3 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 2b11498a32..07e0f14c11 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -1088,7 +1088,7 @@ public class WorkflowExecuteRunnable implements Runnable {
     }
 
     /**
-     * encapsulation task
+     * encapsulation task, this method will only create a new task instance, the return task instance will not contain id.
      *
      * @param processInstance process instance
      * @param taskNode        taskNode
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
index 2e939ee332..cc7f39e402 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueue.java
@@ -29,6 +29,8 @@ import java.util.PriorityQueue;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
+import com.google.common.base.Preconditions;
+
 /**
  * Task instances priority queue implementation
  * All the task instances are in the same process instance.
@@ -43,7 +45,7 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      * queue
      */
     private final PriorityQueue<TaskInstance> queue = new PriorityQueue<>(QUEUE_MAX_SIZE, new TaskInfoComparator());
-    private final Set<Integer> taskInstanceIdSet = Collections.synchronizedSet(new HashSet<>());
+    private final Set<String> taskInstanceIdentifySet = Collections.synchronizedSet(new HashSet<>());
 
     /**
      * put task instance to priority queue
@@ -53,8 +55,9 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      */
     @Override
     public void put(TaskInstance taskInstance) throws TaskPriorityQueueException {
+        Preconditions.checkNotNull(taskInstance);
         queue.add(taskInstance);
-        taskInstanceIdSet.add(taskInstance.getId());
+        taskInstanceIdentifySet.add(getTaskInstanceIdentify(taskInstance));
     }
 
     /**
@@ -67,7 +70,7 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
     public TaskInstance take() throws TaskPriorityQueueException {
         TaskInstance taskInstance = queue.poll();
         if (taskInstance != null) {
-            taskInstanceIdSet.remove(taskInstance.getId());
+            taskInstanceIdentifySet.remove(getTaskInstanceIdentify(taskInstance));
         }
         return taskInstance;
     }
@@ -114,7 +117,7 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      */
     public void clear() {
         queue.clear();
-        taskInstanceIdSet.clear();
+        taskInstanceIdentifySet.clear();
     }
 
     /**
@@ -124,11 +127,8 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      * @return true is contains
      */
     public boolean contains(TaskInstance taskInstance) {
-        return this.contains(taskInstance.getId());
-    }
-
-    public boolean contains(int taskInstanceId) {
-        return taskInstanceIdSet.contains(taskInstanceId);
+        Preconditions.checkNotNull(taskInstance);
+        return taskInstanceIdentifySet.contains(getTaskInstanceIdentify(taskInstance));
     }
 
     /**
@@ -138,6 +138,8 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
      * @return true if remove success
      */
     public boolean remove(TaskInstance taskInstance) {
+        Preconditions.checkNotNull(taskInstance);
+        taskInstanceIdentifySet.remove(getTaskInstanceIdentify(taskInstance));
         return queue.remove(taskInstance);
     }
 
@@ -150,10 +152,20 @@ public class PeerTaskInstancePriorityQueue implements TaskPriorityQueue<TaskInst
         return queue.iterator();
     }
 
+    // since the task instance will not contain taskInstanceId until insert into database
+    // So we use processInstanceId + taskCode + version to identify a taskInstance.
+    private String getTaskInstanceIdentify(TaskInstance taskInstance) {
+        return String.join(
+                String.valueOf(taskInstance.getProcessInstanceId()),
+                String.valueOf(taskInstance.getTaskCode()),
+                String.valueOf(taskInstance.getTaskDefinitionVersion())
+                , "-");
+    }
+
     /**
      * TaskInfoComparator
      */
-    private class TaskInfoComparator implements Comparator<TaskInstance> {
+    private static class TaskInfoComparator implements Comparator<TaskInstance> {
 
         /**
          * compare o1 o2
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java
index 67e40d1189..0daf8bf66a 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/queue/PeerTaskInstancePriorityQueueTest.java
@@ -36,6 +36,8 @@ public class PeerTaskInstancePriorityQueueTest {
         queue.put(taskInstanceHigPriority);
         queue.put(taskInstanceMediumPriority);
         Assert.assertEquals(2, queue.size());
+        Assert.assertTrue(queue.contains(taskInstanceHigPriority));
+        Assert.assertTrue(queue.contains(taskInstanceMediumPriority));
     }
 
     @Test
@@ -108,6 +110,9 @@ public class PeerTaskInstancePriorityQueueTest {
         TaskInstance taskInstanceMediumPriority = createTaskInstance("medium", Priority.MEDIUM, 1);
         queue.put(taskInstanceMediumPriority);
         Assert.assertTrue(queue.contains(taskInstanceMediumPriority));
+        TaskInstance taskInstance2 = createTaskInstance("medium2", Priority.MEDIUM, 1);
+        taskInstance2.setProcessInstanceId(2);
+        Assert.assertFalse(queue.contains(taskInstance2));
     }
 
     @Test
@@ -118,6 +123,7 @@ public class PeerTaskInstancePriorityQueueTest {
         int peekBeforeLength = queue.size();
         queue.remove(taskInstanceMediumPriority);
         Assert.assertNotEquals(peekBeforeLength, queue.size());
+        Assert.assertFalse(queue.contains(taskInstanceMediumPriority));
     }
 
     /**


[dolphinscheduler] 28/29: Fix worker cannot shutdown due to resource close failed or heart beat check failed (#10979)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit f3250bf5fa6c33cacb92bc31cdcb1d734e8cc27b
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Fri Jul 15 20:06:53 2022 +0800

    Fix worker cannot shutdown due to resource close failed or heart beat check failed (#10979)
    
    * Use try-with-resource to close resource, and add heart error threshold to avoid worker cannot close due to heart beat check failed
    
    * Move heartbeat error threshold to applicaiton.yml
    
    (cherry picked from commit 2be1d4bf0add2485d4bf322aef7bbaeabfc223de)
---
 .../apache/dolphinscheduler/common/Constants.java  |  1 +
 .../common/thread/ThreadUtils.java                 | 14 +++++---
 .../server/master/MasterServer.java                | 36 +++++++++----------
 .../server/master/config/MasterConfig.java         |  7 ++++
 .../master/registry/MasterRegistryClient.java      |  8 +++--
 .../master/runner/MasterSchedulerBootstrap.java    |  3 +-
 .../src/main/resources/application.yaml            |  2 ++
 .../server/registry/HeartBeatTask.java             | 19 ++++++++--
 .../service/bean/SpringApplicationContext.java     |  3 +-
 .../apache/dolphinscheduler/StandaloneServer.java  | 32 +++++++++++++++--
 .../src/main/resources/application.yaml            |  4 +++
 .../server/worker/WorkerServer.java                | 41 +++++++---------------
 .../server/worker/config/WorkerConfig.java         |  7 ++++
 .../worker/registry/WorkerRegistryClient.java      | 36 +++++++++++--------
 .../src/main/resources/application.yaml            |  2 ++
 15 files changed, 137 insertions(+), 78 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index 5b813b22c7..b4397f007a 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -50,6 +50,7 @@ public final class Constants {
     public static final String REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS = "/lock/failover/masters";
     public static final String REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS = "/lock/failover/workers";
     public static final String REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "/lock/failover/startup-masters";
+
     public static final String FORMAT_SS = "%s%s";
     public static final String FORMAT_S_S = "%s/%s";
     public static final String FOLDER_SEPARATOR = "/";
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
index 5c8020b7cd..f4f2a17bc7 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
@@ -21,12 +21,18 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 public class ThreadUtils {
+
+    private static final Logger logger = LoggerFactory.getLogger(ThreadUtils.class);
+
     /**
      * Wrapper over newDaemonFixedThreadExecutor.
      *
@@ -35,10 +41,7 @@ public class ThreadUtils {
      * @return ExecutorService
      */
     public static ExecutorService newDaemonFixedThreadExecutor(String threadName, int threadsNum) {
-        ThreadFactory threadFactory = new ThreadFactoryBuilder()
-            .setDaemon(true)
-            .setNameFormat(threadName)
-            .build();
+        ThreadFactory threadFactory = new ThreadFactoryBuilder().setDaemon(true).setNameFormat(threadName).build();
         return Executors.newFixedThreadPool(threadsNum, threadFactory);
     }
 
@@ -48,8 +51,9 @@ public class ThreadUtils {
     public static void sleep(final long millis) {
         try {
             Thread.sleep(millis);
-        } catch (final InterruptedException ignore) {
+        } catch (final InterruptedException interruptedException) {
             Thread.currentThread().interrupt();
+            logger.error("Current thread sleep error", interruptedException);
         }
     }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 66b258d5b2..6711ff3289 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -115,31 +115,27 @@ public class MasterServer implements IStoppable {
      * @param cause close cause
      */
     public void close(String cause) {
-
-        try {
-            // set stop signal is true
-            // execute only once
-            if (!Stopper.stop()) {
-                logger.warn("MasterServer is already stopped, current cause: {}", cause);
-                return;
-            }
+        // set stop signal is true
+        // execute only once
+        if (!Stopper.stop()) {
+            logger.warn("MasterServer is already stopped, current cause: {}", cause);
+            return;
+        }
+        // thread sleep 3 seconds for thread quietly stop
+        ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
+        try (MasterSchedulerBootstrap closedSchedulerBootstrap = masterSchedulerBootstrap;
+             MasterRPCServer closedRpcServer = masterRPCServer;
+             MasterRegistryClient closedMasterRegistryClient = masterRegistryClient;
+             // close spring Context and will invoke method with @PreDestroy annotation to destroy beans.
+             // like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc
+             SpringApplicationContext closedSpringContext = springApplicationContext) {
 
             logger.info("Master server is stopping, current cause : {}", cause);
-
-            // thread sleep 3 seconds for thread quietly stop
-            ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
-            // close
-            this.masterSchedulerBootstrap.close();
-            this.masterRPCServer.close();
-            this.masterRegistryClient.closeRegistry();
-            // close spring Context and will invoke method with @PreDestroy annotation to destroy beans.
-            // like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc
-            springApplicationContext.close();
-
-            logger.info("MasterServer stopped, current cause: {}", cause);
         } catch (Exception e) {
             logger.error("MasterServer stop failed, current cause: {}", cause, e);
+            return;
         }
+        logger.info("MasterServer stopped, current cause: {}", cause);
     }
 
     @Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
index fe51e20984..7f6f124164 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
@@ -67,6 +67,10 @@ public class MasterConfig implements Validator {
      * Master heart beat task execute interval.
      */
     private Duration heartbeatInterval = Duration.ofSeconds(10);
+    /**
+     * Master heart beat task error threshold, if the continuous error count exceed this count, the master will close.
+     */
+    private int heartbeatErrorThreshold = 5;
     /**
      * task submit max retry times.
      */
@@ -129,6 +133,9 @@ public class MasterConfig implements Validator {
         if (masterConfig.getMaxCpuLoadAvg() <= 0) {
             masterConfig.setMaxCpuLoadAvg(Runtime.getRuntime().availableProcessors() * 2);
         }
+        if (masterConfig.getHeartbeatErrorThreshold() <= 0) {
+            errors.rejectValue("heartbeat-error-threshold", null, "should be a positive value");
+        }
         masterConfig.setMasterAddress(NetUtils.getAddr(masterConfig.getListenPort()));
     }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
index 486c360117..a59b712828 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
@@ -53,7 +53,7 @@ import com.google.common.collect.Sets;
  * <p>When the Master node startup, it will register in registry center. And schedule a {@link HeartBeatTask} to update its metadata in registry.
  */
 @Component
-public class MasterRegistryClient {
+public class MasterRegistryClient implements AutoCloseable {
 
     /**
      * logger
@@ -108,7 +108,8 @@ public class MasterRegistryClient {
         registryClient.setStoppable(stoppable);
     }
 
-    public void closeRegistry() {
+    @Override
+    public void close() {
         // TODO unsubscribe MasterRegistryDataListener
         deregister();
     }
@@ -194,7 +195,8 @@ public class MasterRegistryClient {
                                                         masterConfig.getReservedMemory(),
                                                         Sets.newHashSet(localNodePath),
                                                         Constants.MASTER_TYPE,
-                                                        registryClient);
+                                                        registryClient,
+                                                        masterConfig.getHeartbeatErrorThreshold());
 
         // remove before persist
         registryClient.remove(localNodePath);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
index b013a0e9c4..523e5f839d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
@@ -57,7 +57,7 @@ import org.springframework.stereotype.Service;
  * Master scheduler thread, this thread will consume the commands from database and trigger processInstance executed.
  */
 @Service
-public class MasterSchedulerBootstrap extends BaseDaemonThread {
+public class MasterSchedulerBootstrap extends BaseDaemonThread implements AutoCloseable {
 
     private static final Logger logger = LoggerFactory.getLogger(MasterSchedulerBootstrap.class);
 
@@ -112,6 +112,7 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread {
         logger.info("Master schedule bootstrap started...");
     }
 
+    @Override
     public void close() {
         logger.info("Master schedule bootstrap stopping...");
         logger.info("Master schedule bootstrap stopped...");
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index ff29b19389..b559806d28 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -98,6 +98,8 @@ master:
   host-selector: lower_weight
   # master heartbeat interval
   heartbeat-interval: 10s
+  # Master heart beat task error threshold, if the continuous error count exceed this count, the master will close.
+  heartbeat-error-threshold: 5
   # master commit task retry times
   task-commit-retry-times: 5
   # master commit task interval
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
index 129aaff0de..c84abb4182 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
@@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.utils.HeartBeat;
 import org.apache.dolphinscheduler.service.registry.RegistryClient;
 
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,16 +39,22 @@ public class HeartBeatTask implements Runnable {
     private final String serverType;
     private final HeartBeat heartBeat;
 
+    private final int heartBeatErrorThreshold;
+
+    private final AtomicInteger heartBeatErrorTimes = new AtomicInteger();
+
     public HeartBeatTask(long startupTime,
                          double maxCpuloadAvg,
                          double reservedMemory,
                          Set<String> heartBeatPaths,
                          String serverType,
-                         RegistryClient registryClient) {
+                         RegistryClient registryClient,
+                         int heartBeatErrorThreshold) {
         this.heartBeatPaths = heartBeatPaths;
         this.registryClient = registryClient;
         this.serverType = serverType;
         this.heartBeat = new HeartBeat(startupTime, maxCpuloadAvg, reservedMemory);
+        this.heartBeatErrorThreshold = heartBeatErrorThreshold;
     }
 
     public HeartBeatTask(long startupTime,
@@ -58,13 +65,14 @@ public class HeartBeatTask implements Runnable {
                          String serverType,
                          RegistryClient registryClient,
                          int workerThreadCount,
-                         int workerWaitingTaskCount
-    ) {
+                         int workerWaitingTaskCount,
+                         int heartBeatErrorThreshold) {
         this.heartBeatPaths = heartBeatPaths;
         this.registryClient = registryClient;
         this.workerWaitingTaskCount = workerWaitingTaskCount;
         this.serverType = serverType;
         this.heartBeat = new HeartBeat(startupTime, maxCpuloadAvg, reservedMemory, hostWeight, workerThreadCount);
+        this.heartBeatErrorThreshold = heartBeatErrorThreshold;
     }
 
     public String getHeartBeatInfo() {
@@ -88,8 +96,13 @@ public class HeartBeatTask implements Runnable {
             for (String heartBeatPath : heartBeatPaths) {
                 registryClient.persistEphemeral(heartBeatPath, heartBeat.encodeHeartBeat());
             }
+            heartBeatErrorTimes.set(0);
         } catch (Throwable ex) {
             logger.error("HeartBeat task execute failed", ex);
+            if (heartBeatErrorTimes.incrementAndGet() >= heartBeatErrorThreshold) {
+                registryClient.getStoppable()
+                              .stop("HeartBeat task connect to zk failed too much times: " + heartBeatErrorTimes);
+            }
         }
     }
 }
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
index 61dfcb35d7..5b37b1f72d 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/bean/SpringApplicationContext.java
@@ -24,7 +24,7 @@ import org.springframework.context.support.AbstractApplicationContext;
 import org.springframework.stereotype.Component;
 
 @Component
-public class SpringApplicationContext implements ApplicationContextAware {
+public class SpringApplicationContext implements ApplicationContextAware, AutoCloseable {
 
     private static ApplicationContext applicationContext;
 
@@ -36,6 +36,7 @@ public class SpringApplicationContext implements ApplicationContextAware {
     /**
      * Close this application context, destroying all beans in its bean factory.
      */
+    @Override
     public void close() {
         ((AbstractApplicationContext)applicationContext).close();
     }
diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java
index 3866c53066..728fed8eaa 100644
--- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java
+++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/StandaloneServer.java
@@ -19,15 +19,41 @@ package org.apache.dolphinscheduler;
 
 import org.apache.curator.test.TestingServer;
 
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.event.ApplicationFailedEvent;
+import org.springframework.context.ApplicationEvent;
+import org.springframework.context.ApplicationListener;
+import org.springframework.context.event.ContextClosedEvent;
+
+import lombok.NonNull;
 
 @SpringBootApplication
-public class StandaloneServer {
+public class StandaloneServer implements ApplicationListener<ApplicationEvent> {
+
+    private static final Logger logger = LoggerFactory.getLogger(StandaloneServer.class);
+
+    private static TestingServer zookeeperServer;
 
     public static void main(String[] args) throws Exception {
-        final TestingServer server = new TestingServer(true);
-        System.setProperty("registry.zookeeper.connect-string", server.getConnectString());
+        zookeeperServer = new TestingServer(true);
+        System.setProperty("registry.zookeeper.connect-string", zookeeperServer.getConnectString());
         SpringApplication.run(StandaloneServer.class, args);
     }
+
+    @Override
+    public void onApplicationEvent(@NonNull ApplicationEvent event) {
+        if (event instanceof ApplicationFailedEvent || event instanceof ContextClosedEvent) {
+            try (TestingServer closedServer = zookeeperServer) {
+                // close the zookeeper server
+                logger.info("Receive spring context close event: {}, will closed zookeeper server", event);
+            } catch (IOException e) {
+                logger.error("Close zookeeper server error", e);
+            }
+        }
+    }
 }
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index 9a940b6c0e..8411360275 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -116,6 +116,8 @@ master:
   host-selector: lower_weight
   # master heartbeat interval
   heartbeat-interval: 10s
+  # Master heart beat task error threshold, if the continuous error count exceed this count, the master will close.
+  heartbeat-error-threshold: 5
   # master commit task retry times
   task-commit-retry-times: 5
   # master commit task interval
@@ -137,6 +139,8 @@ worker:
   exec-threads: 10
   # worker heartbeat interval
   heartbeat-interval: 10s
+  # Worker heart beat task error threshold, if the continuous error count exceed this count, the worker will close.
+  heartbeat-error-threshold: 5
   # worker host weight to dispatch tasks, default value 100
   host-weight: 100
   # worker tenant auto create
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
index 0321b76f44..471bc008eb 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
@@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.server.worker;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.IStoppable;
-import org.apache.dolphinscheduler.common.enums.NodeType;
 import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.plugin.task.api.ProcessUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -37,7 +37,6 @@ import org.apache.dolphinscheduler.service.task.TaskPluginManager;
 import org.apache.commons.collections4.CollectionUtils;
 
 import java.util.Collection;
-import java.util.Set;
 
 import javax.annotation.PostConstruct;
 
@@ -111,8 +110,7 @@ public class WorkerServer implements IStoppable {
 
         this.workerRegistryClient.registry();
         this.workerRegistryClient.setRegistryStoppable(this);
-        Set<String> workerZkPaths = this.workerRegistryClient.getWorkerZkPaths();
-        this.workerRegistryClient.handleDeadServer(workerZkPaths, NodeType.WORKER, Constants.DELETE_OP);
+        this.workerRegistryClient.handleDeadServer();
 
         this.workerManagerThread.start();
 
@@ -129,37 +127,24 @@ public class WorkerServer implements IStoppable {
     }
 
     public void close(String cause) {
-        try {
-            // execute only once
-            // set stop signal is true
-            if (!Stopper.stop()) {
-                logger.warn("WorkerServer is already stopped, current cause: {}", cause);
-                return;
-            }
+        if (!Stopper.stop()) {
+            logger.warn("WorkerServer is already stopped, current cause: {}", cause);
+            return;
+        }
+        ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
 
+        try (WorkerRpcServer closedWorkerRpcServer = workerRpcServer;
+             WorkerRegistryClient closedRegistryClient = workerRegistryClient;
+             AlertClientService closedAlertClientService = alertClientService;
+             SpringApplicationContext closedSpringContext = springApplicationContext;) {
             logger.info("Worker server is stopping, current cause : {}", cause);
-
-            try {
-                // thread sleep 3 seconds for thread quitely stop
-                Thread.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
-            } catch (Exception e) {
-                logger.warn("Worker server close wait error", e);
-            }
-
-            // close
-            this.workerRpcServer.close();
-            this.workerRegistryClient.unRegistry();
-            this.alertClientService.close();
-
             // kill running tasks
             this.killAllRunningTasks();
-
-            // close the application context
-            this.springApplicationContext.close();
-            logger.info("Worker server stopped, current cause: {}", cause);
         } catch (Exception e) {
             logger.error("Worker server stop failed, current cause: {}", cause, e);
+            return;
         }
+        logger.info("Worker server stopped, current cause: {}", cause);
     }
 
     @Override
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
index aab162d6d4..2367975715 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
@@ -40,6 +40,10 @@ public class WorkerConfig implements Validator {
     private int listenPort = 1234;
     private int execThreads = 10;
     private Duration heartbeatInterval = Duration.ofSeconds(10);
+    /**
+     * Worker heart beat task error threshold, if the continuous error count exceed this count, the worker will close.
+     */
+    private int heartbeatErrorThreshold = 5;
     private int hostWeight = 100;
     private boolean tenantAutoCreate = true;
     private boolean tenantDistributedUser = false;
@@ -70,6 +74,9 @@ public class WorkerConfig implements Validator {
         if (workerConfig.getMaxCpuLoadAvg() <= 0) {
             workerConfig.setMaxCpuLoadAvg(Runtime.getRuntime().availableProcessors() * 2);
         }
+        if (workerConfig.getHeartbeatErrorThreshold() <= 0) {
+            errors.rejectValue("heartbeat-error-threshold", null, "should be a positive value");
+        }
         workerConfig.setWorkerAddress(NetUtils.getAddr(workerConfig.getListenPort()));
     }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
index b33b3ef0d7..99d18acc30 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
@@ -55,7 +55,7 @@ import com.google.common.collect.Sets;
  * worker registry
  */
 @Service
-public class WorkerRegistryClient {
+public class WorkerRegistryClient implements AutoCloseable {
 
     private final Logger logger = LoggerFactory.getLogger(WorkerRegistryClient.class);
 
@@ -102,15 +102,15 @@ public class WorkerRegistryClient {
         long workerHeartbeatInterval = workerConfig.getHeartbeatInterval().getSeconds();
 
         HeartBeatTask heartBeatTask = new HeartBeatTask(startupTime,
-                workerConfig.getMaxCpuLoadAvg(),
-                workerConfig.getReservedMemory(),
-                workerConfig.getHostWeight(),
-                workerZkPaths,
-                Constants.WORKER_TYPE,
-                registryClient,
-                workerConfig.getExecThreads(),
-                workerManagerThread.getThreadPoolQueueSize()
-        );
+                                                        workerConfig.getMaxCpuLoadAvg(),
+                                                        workerConfig.getReservedMemory(),
+                                                        workerConfig.getHostWeight(),
+                                                        workerZkPaths,
+                                                        Constants.WORKER_TYPE,
+                                                        registryClient,
+                                                        workerConfig.getExecThreads(),
+                                                        workerManagerThread.getThreadPoolQueueSize(),
+                                                        workerConfig.getHeartbeatErrorThreshold());
 
         for (String workerZKPath : workerZkPaths) {
             // remove before persist
@@ -148,8 +148,10 @@ public class WorkerRegistryClient {
             logger.error("remove worker zk path exception", ex);
         }
 
-        this.heartBeatExecutor.shutdownNow();
-        logger.info("heartbeat executor shutdown");
+        if (heartBeatExecutor != null) {
+            heartBeatExecutor.shutdownNow();
+            logger.info("Heartbeat executor shutdown");
+        }
 
         registryClient.close();
         logger.info("registry client closed");
@@ -176,8 +178,9 @@ public class WorkerRegistryClient {
         return workerPaths;
     }
 
-    public void handleDeadServer(Set<String> nodeSet, NodeType nodeType, String opType) {
-        registryClient.handleDeadServer(nodeSet, nodeType, opType);
+    public void handleDeadServer() {
+        Set<String> workerZkPaths = getWorkerZkPaths();
+        registryClient.handleDeadServer(workerZkPaths, NodeType.WORKER, Constants.DELETE_OP);
     }
 
     /**
@@ -191,4 +194,9 @@ public class WorkerRegistryClient {
         registryClient.setStoppable(stoppable);
     }
 
+    @Override
+    public void close() throws IOException {
+        unRegistry();
+    }
+
 }
diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml
index d764f9d25b..7d653edc24 100644
--- a/dolphinscheduler-worker/src/main/resources/application.yaml
+++ b/dolphinscheduler-worker/src/main/resources/application.yaml
@@ -60,6 +60,8 @@ worker:
   exec-threads: 100
   # worker heartbeat interval
   heartbeat-interval: 10s
+  # Worker heart beat task error threshold, if the continuous error count exceed this count, the worker will close.
+  heartbeat-error-threshold: 5
   # worker host weight to dispatch tasks, default value 100
   host-weight: 100
   # worker tenant auto create


[dolphinscheduler] 23/29: [Fix-10827] Fix network error cause worker cannot send message to master (#10886)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 9ee20cffefa44079ea7435cd8bd6366820c4e624
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jul 12 14:08:42 2022 +0800

    [Fix-10827] Fix network error cause worker cannot send message to master (#10886)
    
    * Fix network error cause worker cannot send message to master
    
    (cherry picked from commit cade66a9b6b14450c6f778f7e05820b588856a41)
---
 .../server/master/config/MasterConfig.java         |   6 +
 .../master/consumer/TaskPriorityQueueConsumer.java |   8 +-
 .../dispatch/executor/NettyExecutorManager.java    |   2 -
 .../server/master/event/TaskDelayEventHandler.java |   8 +-
 .../event/TaskRejectByWorkerEventHandler.java      |  19 +-
 .../master/event/TaskResultEventHandler.java       |  20 +-
 .../master/event/TaskRunningEventHandler.java      |   8 +-
 .../processor/TaskExecuteResponseProcessor.java    |  17 +-
 .../processor/TaskExecuteRunningProcessor.java     |   6 +-
 .../server/master/processor/queue/TaskEvent.java   |   4 +-
 .../server/master/rpc/MasterRPCServer.java         |   7 +-
 .../master/runner/MasterSchedulerBootstrap.java    |   5 +-
 .../master/runner/StateWheelExecuteThread.java     |   2 -
 .../master/dispatch/ExecutionContextTestUtils.java |  16 +-
 .../master/dispatch/ExecutorDispatcherTest.java    |   6 +-
 .../executor/NettyExecutorManagerTest.java         |  25 +-
 .../master/processor/TaskAckProcessorTest.java     |  20 +-
 .../processor/queue/TaskResponseServiceTest.java   |  47 ++--
 .../remote/command/BaseCommand.java                |  57 +++++
 .../remote/command/CommandHeader.java              |  47 +---
 .../remote/command/CommandType.java                |  26 +--
 .../remote/command/HostUpdateCommand.java          |   3 -
 .../remote/command/StateEventResponseCommand.java  |   2 +-
 ...equestCommand.java => TaskDispatchCommand.java} |  24 +-
 ...eAckCommand.java => TaskExecuteAckCommand.java} |  48 ++--
 .../remote/command/TaskExecuteResponseCommand.java | 212 -----------------
 ...kCommand.java => TaskExecuteResultCommand.java} |  97 +++++---
 ...mand.java => TaskExecuteRunningAckMessage.java} |   6 +-
 .../remote/command/TaskExecuteRunningCommand.java  | 101 ++------
 ...pdateCommand.java => TaskRejectAckCommand.java} |  53 ++---
 ...skRecallCommand.java => TaskRejectCommand.java} |   0
 .../apache/dolphinscheduler/remote/utils/Host.java |   3 +-
 .../server/worker/WorkerServer.java                |  18 +-
 .../server/worker/cache/ResponseCache.java         |  98 --------
 .../server/worker/config/WorkerConfig.java         |   8 +-
 .../server/worker/message/MessageRetryRunner.java  | 139 +++++++++++
 .../server/worker/message/MessageSender.java       |  49 ++--
 .../message/TaskExecuteResultMessageSender.java    |  70 ++++++
 .../message/TaskExecuteRunningMessageSender.java   |  67 ++++++
 .../worker/message/TaskRejectMessageSender.java    |  59 +++++
 .../worker/processor/HostUpdateProcessor.java      |  13 +-
 .../worker/processor/TaskCallbackService.java      | 258 ---------------------
 ...teProcessor.java => TaskDispatchProcessor.java} |  60 +++--
 .../processor/TaskExecuteResponseAckProcessor.java |  71 ------
 .../processor/TaskExecuteResultAckProcessor.java   |  83 +++++++
 .../processor/TaskExecuteRunningAckProcessor.java  |  17 +-
 .../server/worker/processor/TaskKillProcessor.java |  49 ++--
 .../worker/processor/TaskRejectAckProcessor.java   |   0
 .../server/worker/rpc/WorkerMessageSender.java     |  92 ++++++++
 .../server/worker/rpc/WorkerRpcClient.java         |  70 ++++++
 .../worker/{prc => rpc}/WorkerRpcServer.java       |  20 +-
 .../worker/runner/RetryReportTaskStatusThread.java | 117 ----------
 .../server/worker/runner/TaskExecuteThread.java    |  82 ++++---
 .../server/worker/runner/WorkerExecService.java    |   6 +-
 .../server/worker/runner/WorkerManagerThread.java  |  34 +--
 ...sorTest.java => TaskDispatchProcessorTest.java} |  92 ++++----
 .../worker/runner/TaskExecuteThreadTest.java       |  12 +-
 57 files changed, 1169 insertions(+), 1320 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
index 0494b7dd00..fe51e20984 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.config;
 
+import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelector;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
@@ -82,6 +83,10 @@ public class MasterConfig implements Validator {
     private double reservedMemory = 0.3;
     private Duration failoverInterval = Duration.ofMinutes(10);
     private boolean killYarnJobWhenTaskFailover = true;
+    /**
+     * ip:listenPort
+     */
+    private String masterAddress;
 
     @Override
     public boolean supports(Class<?> clazz) {
@@ -124,5 +129,6 @@ public class MasterConfig implements Validator {
         if (masterConfig.getMaxCpuLoadAvg() <= 0) {
             masterConfig.setMaxCpuLoadAvg(Runtime.getRuntime().availableProcessors() * 2);
         }
+        masterConfig.setMasterAddress(NetUtils.getAddr(masterConfig.getListenPort()));
     }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index 82198e942a..9c7d048ff8 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -24,7 +24,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.remote.command.Command;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
+import org.apache.dolphinscheduler.remote.command.TaskDispatchCommand;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher;
@@ -241,7 +241,11 @@ public class TaskPriorityQueueConsumer extends BaseDaemonThread {
     }
 
     private Command toCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(taskExecutionContext);
+        // todo: we didn't set the host here, since right now we didn't need to retry this message.
+        TaskDispatchCommand requestCommand = new TaskDispatchCommand(taskExecutionContext,
+                                                                     masterConfig.getMasterAddress(),
+                                                                     taskExecutionContext.getHost(),
+                                                                     System.currentTimeMillis());
         return requestCommand.convert2Command();
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
index 0cc2d4b8f7..06c78773be 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManager.java
@@ -83,8 +83,6 @@ public class NettyExecutorManager extends AbstractExecutorManager<Boolean> {
 
     @PostConstruct
     public void init() {
-        this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE, taskExecuteResponseProcessor);
-        this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RUNNING, taskExecuteRunningProcessor);
         this.nettyRemotingClient.registerProcessor(CommandType.TASK_KILL_RESPONSE, taskKillResponseProcessor);
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java
index 8e2dcced23..25fa88dd00 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java
@@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckMessage;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
@@ -108,9 +108,9 @@ public class TaskDelayEventHandler implements TaskEventHandler {
 
     private void sendAckToWorker(TaskEvent taskEvent) {
         // If event handle success, send ack to worker to otherwise the worker will retry this event
-        TaskExecuteRunningAckCommand taskExecuteRunningAckCommand =
-            new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-        taskEvent.getChannel().writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
+        TaskExecuteRunningAckMessage taskExecuteRunningAckMessage =
+            new TaskExecuteRunningAckMessage(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        taskEvent.getChannel().writeAndFlush(taskExecuteRunningAckMessage.convert2Command());
     }
 
     @Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java
index d09a8364ca..ac4dab50f2 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java
@@ -20,8 +20,9 @@ package org.apache.dolphinscheduler.server.master.event;
 import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.TaskRecallAckCommand;
+import org.apache.dolphinscheduler.remote.command.TaskRejectAckCommand;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
@@ -34,13 +35,16 @@ public class TaskRejectByWorkerEventHandler implements TaskEventHandler {
     @Autowired
     private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
 
+    @Autowired
+    private MasterConfig masterConfig;
+
     @Override
     public void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError {
         int taskInstanceId = taskEvent.getTaskInstanceId();
         int processInstanceId = taskEvent.getProcessInstanceId();
 
-        WorkflowExecuteRunnable workflowExecuteRunnable =
-            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        WorkflowExecuteRunnable workflowExecuteRunnable = this.processInstanceExecCacheManager.getByProcessInstanceId(
+            processInstanceId);
         if (workflowExecuteRunnable == null) {
             sendAckToWorker(taskEvent);
             throw new TaskEventHandleError(
@@ -65,9 +69,12 @@ public class TaskRejectByWorkerEventHandler implements TaskEventHandler {
     }
 
     public void sendAckToWorker(TaskEvent taskEvent) {
-        TaskRecallAckCommand taskRecallAckCommand =
-            new TaskRecallAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-        taskEvent.getChannel().writeAndFlush(taskRecallAckCommand.convert2Command());
+        TaskRejectAckCommand taskRejectAckMessage = new TaskRejectAckCommand(ExecutionStatus.SUCCESS.getCode(),
+                                                                             taskEvent.getTaskInstanceId(),
+                                                                             masterConfig.getMasterAddress(),
+                                                                             taskEvent.getWorkerAddress(),
+                                                                             System.currentTimeMillis());
+        taskEvent.getChannel().writeAndFlush(taskRejectAckMessage.convert2Command());
     }
 
     @Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java
index 67df03682f..0373baeee1 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java
@@ -22,8 +22,9 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
@@ -50,13 +51,16 @@ public class TaskResultEventHandler implements TaskEventHandler {
     @Autowired
     private ProcessService processService;
 
+    @Autowired
+    private MasterConfig masterConfig;
+
     @Override
     public void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError, TaskEventHandleException {
         int taskInstanceId = taskEvent.getTaskInstanceId();
         int processInstanceId = taskEvent.getProcessInstanceId();
 
-        WorkflowExecuteRunnable workflowExecuteRunnable =
-            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        WorkflowExecuteRunnable workflowExecuteRunnable = this.processInstanceExecCacheManager.getByProcessInstanceId(
+            processInstanceId);
         if (workflowExecuteRunnable == null) {
             sendAckToWorker(taskEvent);
             throw new TaskEventHandleError(
@@ -105,9 +109,13 @@ public class TaskResultEventHandler implements TaskEventHandler {
     }
 
     public void sendAckToWorker(TaskEvent taskEvent) {
-        TaskExecuteResponseAckCommand taskExecuteResponseAckCommand =
-            new TaskExecuteResponseAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-        taskEvent.getChannel().writeAndFlush(taskExecuteResponseAckCommand.convert2Command());
+        // we didn't set the receiver address, since the ack doen's need to retry
+        TaskExecuteAckCommand taskExecuteAckMessage = new TaskExecuteAckCommand(ExecutionStatus.SUCCESS.getCode(),
+                                                                                taskEvent.getTaskInstanceId(),
+                                                                                masterConfig.getMasterAddress(),
+                                                                                taskEvent.getWorkerAddress(),
+                                                                                System.currentTimeMillis());
+        taskEvent.getChannel().writeAndFlush(taskExecuteAckMessage.convert2Command());
     }
 
     @Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java
index 31152973a2..d271839596 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java
@@ -22,7 +22,7 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckMessage;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
@@ -106,9 +106,9 @@ public class TaskRunningEventHandler implements TaskEventHandler {
 
     private void sendAckToWorker(TaskEvent taskEvent) {
         // If event handle success, send ack to worker to otherwise the worker will retry this event
-        TaskExecuteRunningAckCommand taskExecuteRunningAckCommand =
-            new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-        taskEvent.getChannel().writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
+        TaskExecuteRunningAckMessage taskExecuteRunningAckMessage =
+            new TaskExecuteRunningAckMessage(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        taskEvent.getChannel().writeAndFlush(taskExecuteRunningAckMessage.convert2Command());
     }
 
     @Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
index b17def873c..7f438e1d9c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
@@ -21,7 +21,7 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteResultCommand;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService;
@@ -55,15 +55,18 @@ public class TaskExecuteResponseProcessor implements NettyRequestProcessor {
      */
     @Override
     public void process(Channel channel, Command command) {
-        Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType()));
+        Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESULT == command.getType(),
+                                    String.format("invalid command type : %s", command.getType()));
 
-        TaskExecuteResponseCommand taskExecuteResponseCommand = JSONUtils.parseObject(command.getBody(), TaskExecuteResponseCommand.class);
-        TaskEvent taskResponseEvent = TaskEvent.newResultEvent(taskExecuteResponseCommand, channel);
+        TaskExecuteResultCommand taskExecuteResultMessage = JSONUtils.parseObject(command.getBody(),
+                                                                                  TaskExecuteResultCommand.class);
+        TaskEvent taskResultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, channel);
         try {
-            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskResponseEvent.getProcessInstanceId(), taskResponseEvent.getTaskInstanceId());
-            logger.info("Received task execute response, event: {}", taskResponseEvent);
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskResultEvent.getProcessInstanceId(),
+                                                        taskResultEvent.getTaskInstanceId());
+            logger.info("Received task execute result, event: {}", taskResultEvent);
 
-            taskEventService.addEvent(taskResponseEvent);
+            taskEventService.addEvent(taskResultEvent);
         } finally {
             LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java
index c5ddc3d77a..96ff1ca405 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java
@@ -54,10 +54,10 @@ public class TaskExecuteRunningProcessor implements NettyRequestProcessor {
     @Override
     public void process(Channel channel, Command command) {
         Preconditions.checkArgument(CommandType.TASK_EXECUTE_RUNNING == command.getType(), String.format("invalid command type : %s", command.getType()));
-        TaskExecuteRunningCommand taskExecuteRunningCommand = JSONUtils.parseObject(command.getBody(), TaskExecuteRunningCommand.class);
-        logger.info("taskExecuteRunningCommand: {}", taskExecuteRunningCommand);
+        TaskExecuteRunningCommand taskExecuteRunningMessage = JSONUtils.parseObject(command.getBody(), TaskExecuteRunningCommand.class);
+        logger.info("taskExecuteRunningCommand: {}", taskExecuteRunningMessage);
 
-        TaskEvent taskEvent = TaskEvent.newRunningEvent(taskExecuteRunningCommand, channel);
+        TaskEvent taskEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, channel);
         taskEventService.addEvent(taskEvent);
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
index 2eb60ce8b9..7db0bc6696 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
@@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.master.processor.queue;
 
 import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteResultCommand;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningCommand;
 import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
 
@@ -119,7 +119,7 @@ public class TaskEvent {
         return event;
     }
 
-    public static TaskEvent newResultEvent(TaskExecuteResponseCommand command, Channel channel) {
+    public static TaskEvent newResultEvent(TaskExecuteResultCommand command, Channel channel) {
         TaskEvent event = new TaskEvent();
         event.setProcessInstanceId(command.getProcessInstanceId());
         event.setTaskInstanceId(command.getTaskInstanceId());
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java
index e0b65dd077..24c3530462 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterRPCServer.java
@@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.server.master.processor.TaskEventProcessor;
 import org.apache.dolphinscheduler.server.master.processor.TaskExecuteResponseProcessor;
 import org.apache.dolphinscheduler.server.master.processor.TaskExecuteRunningProcessor;
 import org.apache.dolphinscheduler.server.master.processor.TaskKillResponseProcessor;
-import org.apache.dolphinscheduler.server.master.processor.TaskRecallProcessor;
 
 import javax.annotation.PostConstruct;
 
@@ -68,9 +67,6 @@ public class MasterRPCServer implements AutoCloseable {
     @Autowired
     private TaskKillResponseProcessor taskKillResponseProcessor;
 
-    @Autowired
-    private TaskRecallProcessor taskRecallProcessor;
-
     @Autowired
     private LoggerRequestProcessor loggerRequestProcessor;
 
@@ -80,14 +76,13 @@ public class MasterRPCServer implements AutoCloseable {
         NettyServerConfig serverConfig = new NettyServerConfig();
         serverConfig.setListenPort(masterConfig.getListenPort());
         this.nettyRemotingServer = new NettyRemotingServer(serverConfig);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE, taskExecuteResponseProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RUNNING, taskExecuteRunningProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESULT, taskExecuteResponseProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_RESPONSE, taskKillResponseProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.STATE_EVENT_REQUEST, stateEventProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_FORCE_STATE_EVENT_REQUEST, taskEventProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_WAKEUP_EVENT_REQUEST, taskEventProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.CACHE_EXPIRE, cacheProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_RECALL, taskRecallProcessor);
 
         // logger server
         this.nettyRemotingServer.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, loggerRequestProcessor);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
index b4c72209db..b013a0e9c4 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
@@ -30,7 +30,9 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEvent;
 import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEventType;
 import org.apache.dolphinscheduler.server.master.exception.MasterException;
 import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
 import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
@@ -156,8 +158,7 @@ public class MasterSchedulerBootstrap extends BaseDaemonThread {
                                                                                                nettyExecutorManager,
                                                                                                processAlertManager,
                                                                                                masterConfig,
-                                                                                               stateWheelExecuteThread,
-                                                                                               curingGlobalParamsService);
+                                                                                               stateWheelExecuteThread);
                         processInstanceExecCacheManager.cache(processInstance.getId(), workflowRunnable);
                         workflowEventQueue.addEvent(new WorkflowEvent(WorkflowEventType.START_WORKFLOW,
                                                                       processInstance.getId()));
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 843c1b246c..0d1b3a423a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -126,8 +126,6 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
         boolean removeFlag = processInstanceTimeoutCheckList.remove(processInstanceId);
         if (removeFlag) {
             logger.info("Success remove workflow instance from timeout check list");
-        } else {
-            logger.warn("Failed to remove workflow instance from timeout check list");
         }
     }
 
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
index 55dd236f71..7252f747ba 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
@@ -24,12 +24,11 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.remote.command.Command;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
+import org.apache.dolphinscheduler.remote.command.TaskDispatchCommand;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
 import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
 import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
-import org.apache.dolphinscheduler.spi.utils.JSONUtils;
 
 import org.mockito.Mockito;
 
@@ -44,12 +43,15 @@ public class ExecutionContextTestUtils {
         processInstance.setCommandType(CommandType.COMPLEMENT_DATA);
         taskInstance.setProcessInstance(processInstance);
         TaskExecutionContext context = TaskExecutionContextBuilder.get()
-                .buildTaskInstanceRelatedInfo(taskInstance)
-                .buildProcessInstanceRelatedInfo(processInstance)
-                .buildProcessDefinitionRelatedInfo(processDefinition)
-                .create();
+                                                                  .buildTaskInstanceRelatedInfo(taskInstance)
+                                                                  .buildProcessInstanceRelatedInfo(processInstance)
+                                                                  .buildProcessDefinitionRelatedInfo(processDefinition)
+                                                                  .create();
 
-        TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(context);
+        TaskDispatchCommand requestCommand = new TaskDispatchCommand(context,
+                                                                     "127.0.0.1:5678",
+                                                                     "127.0.0.1:5678",
+                                                                     System.currentTimeMillis());
         Command command = requestCommand.convert2Command();
 
         ExecutionContext executionContext = new ExecutionContext(command, ExecutorType.WORKER, taskInstance);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
index 4bfc3ca0a6..fa80e8344a 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
@@ -18,11 +18,12 @@
 package org.apache.dolphinscheduler.server.master.dispatch;
 
 import org.apache.dolphinscheduler.remote.NettyRemotingServer;
+import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
 import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
-import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskDispatchProcessor;
 import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient;
 
 import org.junit.Ignore;
@@ -60,7 +61,8 @@ public class ExecutorDispatcherTest {
         final NettyServerConfig serverConfig = new NettyServerConfig();
         serverConfig.setListenPort(port);
         NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig);
-        nettyRemotingServer.registerProcessor(org.apache.dolphinscheduler.remote.command.CommandType.TASK_EXECUTE_REQUEST, Mockito.mock(TaskExecuteProcessor.class));
+        nettyRemotingServer.registerProcessor(CommandType.TASK_DISPATCH_REQUEST, Mockito.mock(
+            TaskDispatchProcessor.class));
         nettyRemotingServer.start();
         //
         workerConfig.setListenPort(port);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
index 1ee890faf2..b69ab890f9 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
@@ -25,15 +25,14 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.remote.NettyRemotingServer;
 import org.apache.dolphinscheduler.remote.command.Command;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
+import org.apache.dolphinscheduler.remote.command.TaskDispatchCommand;
 import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
 import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
 import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
 import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException;
-import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor;
-import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.apache.dolphinscheduler.server.worker.processor.TaskDispatchProcessor;
 
 import org.junit.Assert;
 import org.junit.Ignore;
@@ -49,16 +48,15 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 @RunWith(SpringJUnit4ClassRunner.class)
 @Ignore
 public class NettyExecutorManagerTest {
-
     @Autowired
     private NettyExecutorManager nettyExecutorManager;
-
     @Test
     public void testExecute() throws ExecuteException {
         final NettyServerConfig serverConfig = new NettyServerConfig();
         serverConfig.setListenPort(30000);
         NettyRemotingServer nettyRemotingServer = new NettyRemotingServer(serverConfig);
-        nettyRemotingServer.registerProcessor(org.apache.dolphinscheduler.remote.command.CommandType.TASK_EXECUTE_REQUEST, new TaskExecuteProcessor());
+        nettyRemotingServer.registerProcessor(org.apache.dolphinscheduler.remote.command.CommandType.TASK_DISPATCH_REQUEST,
+                                              new TaskDispatchProcessor());
         nettyRemotingServer.start();
         TaskInstance taskInstance = Mockito.mock(TaskInstance.class);
         ProcessDefinition processDefinition = Mockito.mock(ProcessDefinition.class);
@@ -66,10 +64,10 @@ public class NettyExecutorManagerTest {
         processInstance.setCommandType(CommandType.COMPLEMENT_DATA);
         taskInstance.setProcessInstance(processInstance);
         TaskExecutionContext context = TaskExecutionContextBuilder.get()
-                .buildTaskInstanceRelatedInfo(taskInstance)
-                .buildProcessInstanceRelatedInfo(processInstance)
-                .buildProcessDefinitionRelatedInfo(processDefinition)
-                .create();
+                                                                  .buildTaskInstanceRelatedInfo(taskInstance)
+                                                                  .buildProcessInstanceRelatedInfo(processInstance)
+                                                                  .buildProcessDefinitionRelatedInfo(processDefinition)
+                                                                  .create();
         ExecutionContext executionContext = new ExecutionContext(toCommand(context), ExecutorType.WORKER, taskInstance);
         executionContext.setHost(Host.of(NetUtils.getAddr(serverConfig.getListenPort())));
         Boolean execute = nettyExecutorManager.execute(executionContext);
@@ -94,10 +92,11 @@ public class NettyExecutorManagerTest {
         nettyExecutorManager.execute(executionContext);
 
     }
-
     private Command toCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand();
-        requestCommand.setTaskExecutionContext(taskExecutionContext);
+        TaskDispatchCommand requestCommand = new TaskDispatchCommand(taskExecutionContext,
+                                                                     "127.0.0.1:5678",
+                                                                     "127.0.0.1:1234",
+                                                                     System.currentTimeMillis());
         return requestCommand.convert2Command();
     }
 }
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java
index fc320ab493..8c15cd9cbe 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/TaskAckProcessorTest.java
@@ -44,7 +44,7 @@ public class TaskAckProcessorTest {
     private TaskExecuteRunningProcessor taskExecuteRunningProcessor;
     private TaskEventService taskEventService;
     private ProcessService processService;
-    private TaskExecuteRunningCommand taskExecuteRunningCommand;
+    private TaskExecuteRunningCommand taskExecuteRunningMessage;
     private TaskEvent taskResponseEvent;
     private Channel channel;
 
@@ -63,14 +63,16 @@ public class TaskAckProcessorTest {
         channel = PowerMockito.mock(Channel.class);
         taskResponseEvent = PowerMockito.mock(TaskEvent.class);
 
-        taskExecuteRunningCommand = new TaskExecuteRunningCommand();
-        taskExecuteRunningCommand.setStatus(1);
-        taskExecuteRunningCommand.setExecutePath("/dolphinscheduler/worker");
-        taskExecuteRunningCommand.setHost("localhost");
-        taskExecuteRunningCommand.setLogPath("/temp/worker.log");
-        taskExecuteRunningCommand.setStartTime(new Date());
-        taskExecuteRunningCommand.setTaskInstanceId(1);
-        taskExecuteRunningCommand.setProcessInstanceId(1);
+        taskExecuteRunningMessage = new TaskExecuteRunningCommand("127.0.0.1:5678",
+                                                                  " 127.0.0.1:1234",
+                                                                  System.currentTimeMillis());
+        taskExecuteRunningMessage.setStatus(1);
+        taskExecuteRunningMessage.setExecutePath("/dolphinscheduler/worker");
+        taskExecuteRunningMessage.setHost("localhost");
+        taskExecuteRunningMessage.setLogPath("/temp/worker.log");
+        taskExecuteRunningMessage.setStartTime(new Date());
+        taskExecuteRunningMessage.setTaskInstanceId(1);
+        taskExecuteRunningMessage.setProcessInstanceId(1);
     }
 
     @Test
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
index c5b00db12c..3854ad77b0 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
@@ -17,9 +17,10 @@
 
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
+import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteResultCommand;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningCommand;
 import org.apache.dolphinscheduler.server.master.cache.impl.ProcessInstanceExecCacheManagerImpl;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
@@ -76,26 +77,30 @@ public class TaskResponseServiceTest {
 
         Mockito.when(channel.remoteAddress()).thenReturn(InetSocketAddress.createUnresolved("127.0.0.1", 1234));
 
-        TaskExecuteRunningCommand taskExecuteRunningCommand = new TaskExecuteRunningCommand();
-        taskExecuteRunningCommand.setProcessId(1);
-        taskExecuteRunningCommand.setTaskInstanceId(22);
-        taskExecuteRunningCommand.setStatus(ExecutionStatus.RUNNING_EXECUTION.getCode());
-        taskExecuteRunningCommand.setExecutePath("path");
-        taskExecuteRunningCommand.setLogPath("logPath");
-        taskExecuteRunningCommand.setHost("127.*.*.*");
-        taskExecuteRunningCommand.setStartTime(new Date());
-
-        ackEvent = TaskEvent.newRunningEvent(taskExecuteRunningCommand, channel);
-
-        TaskExecuteResponseCommand taskExecuteResponseCommand = new TaskExecuteResponseCommand();
-        taskExecuteResponseCommand.setProcessInstanceId(1);
-        taskExecuteResponseCommand.setTaskInstanceId(22);
-        taskExecuteResponseCommand.setStatus(ExecutionStatus.SUCCESS.getCode());
-        taskExecuteResponseCommand.setEndTime(new Date());
-        taskExecuteResponseCommand.setVarPool("varPol");
-        taskExecuteResponseCommand.setAppIds("ids");
-        taskExecuteResponseCommand.setProcessId(1);
-        resultEvent = TaskEvent.newResultEvent(taskExecuteResponseCommand, channel);
+        TaskExecuteRunningCommand taskExecuteRunningMessage = new TaskExecuteRunningCommand("127.0.0.1:5678",
+                                                                                            "127.0.0.1:1234",
+                                                                                            System.currentTimeMillis());
+        taskExecuteRunningMessage.setProcessId(1);
+        taskExecuteRunningMessage.setTaskInstanceId(22);
+        taskExecuteRunningMessage.setStatus(ExecutionStatus.RUNNING_EXECUTION.getCode());
+        taskExecuteRunningMessage.setExecutePath("path");
+        taskExecuteRunningMessage.setLogPath("logPath");
+        taskExecuteRunningMessage.setHost("127.*.*.*");
+        taskExecuteRunningMessage.setStartTime(new Date());
+
+        ackEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, channel);
+
+        TaskExecuteResultCommand taskExecuteResultMessage = new TaskExecuteResultCommand(NetUtils.getAddr(1234),
+                                                                                         NetUtils.getAddr(5678),
+                                                                                         System.currentTimeMillis());
+        taskExecuteResultMessage.setProcessInstanceId(1);
+        taskExecuteResultMessage.setTaskInstanceId(22);
+        taskExecuteResultMessage.setStatus(ExecutionStatus.SUCCESS.getCode());
+        taskExecuteResultMessage.setEndTime(new Date());
+        taskExecuteResultMessage.setVarPool("varPol");
+        taskExecuteResultMessage.setAppIds("ids");
+        taskExecuteResultMessage.setProcessId(1);
+        resultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, channel);
 
         taskInstance = new TaskInstance();
         taskInstance.setId(22);
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseCommand.java
new file mode 100644
index 0000000000..8207b5fecc
--- /dev/null
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/BaseCommand.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.remote.command;
+
+import java.io.Serializable;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * This is the base class for rpc message.
+ * <p>
+ * Since we use async mode, the client send a message and will wait the target server
+ * send ack for the message, the client will retry during a while if he doesn't receive an ack.
+ * <p>
+ * When there is a network error, the server cannot send ack to the client by the origin channel,
+ * since the client has closed the channel, so the server need to know the command source.
+ */
+@Data
+@NoArgsConstructor
+public abstract class BaseCommand implements Serializable {
+
+    private static final long serialVersionUID = -1L;
+
+    /**
+     * If the message receiver want to send ack to the sender, need to use this address.
+     */
+    protected String messageSenderAddress;
+
+    /**
+     * The message receiver address.
+     */
+    protected String messageReceiverAddress;
+
+    protected long messageSendTime;
+
+    protected BaseCommand(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) {
+        this.messageSenderAddress = messageSenderAddress;
+        this.messageReceiverAddress = messageReceiverAddress;
+        this.messageSendTime = messageSendTime;
+    }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java
index 9e83a426f9..32eb75dc90 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandHeader.java
@@ -14,13 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.remote.command;
 
 import java.io.Serializable;
 
+import lombok.Data;
+
 /**
  *  command header
  */
+@Data
 public class CommandHeader implements Serializable {
 
     /**
@@ -34,12 +38,12 @@ public class CommandHeader implements Serializable {
     private long opaque;
 
     /**
-     *  context length
+     * context length
      */
     private int contextLength;
 
     /**
-     *  context
+     * context
      */
     private byte[] context;
 
@@ -48,43 +52,4 @@ public class CommandHeader implements Serializable {
      */
     private int bodyLength;
 
-    public int getBodyLength() {
-        return bodyLength;
-    }
-
-    public void setBodyLength(int bodyLength) {
-        this.bodyLength = bodyLength;
-    }
-
-    public byte getType() {
-        return type;
-    }
-
-    public void setType(byte type) {
-        this.type = type;
-    }
-
-    public long getOpaque() {
-        return opaque;
-    }
-
-    public void setOpaque(long opaque) {
-        this.opaque = opaque;
-    }
-
-    public int getContextLength() {
-        return contextLength;
-    }
-
-    public void setContextLength(int contextLength) {
-        this.contextLength = contextLength;
-    }
-
-    public byte[] getContext() {
-        return context;
-    }
-
-    public void setContext(byte[] context) {
-        this.context = context;
-    }
 }
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java
index 5718872bf5..ed4c9ab94c 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/CommandType.java
@@ -64,9 +64,9 @@ public enum CommandType {
     MASTER_RESPONSE,
 
     /**
-     * execute task request
+     * dispatch task request
      */
-    TASK_EXECUTE_REQUEST,
+    TASK_DISPATCH_REQUEST,
 
     /**
      * task execute running, from worker to master
@@ -81,21 +81,15 @@ public enum CommandType {
     /**
      * task execute response, from worker to master
      */
-    TASK_EXECUTE_RESPONSE,
+    TASK_EXECUTE_RESULT,
 
     /**
      * task execute response ack, from master to worker
      */
-    TASK_EXECUTE_RESPONSE_ACK,
+    TASK_EXECUTE_RESULT_ACK,
 
-    /**
-     * kill task
-     */
     TASK_KILL_REQUEST,
 
-    /**
-     * kill task response
-     */
     TASK_KILL_RESPONSE,
 
     /**
@@ -103,24 +97,12 @@ public enum CommandType {
      */
     HEART_BEAT,
 
-    /**
-     * ping
-     */
     PING,
 
-    /**
-     * pong
-     */
     PONG,
 
-    /**
-     * alert send request
-     */
     ALERT_SEND_REQUEST,
 
-    /**
-     * alert send response
-     */
     ALERT_SEND_RESPONSE,
 
     /**
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java
index 4fc752e4b0..d060a8e0f5 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java
@@ -26,9 +26,6 @@ import java.io.Serializable;
  */
 public class HostUpdateCommand implements Serializable {
 
-    /**
-     * task id
-     */
     private int taskInstanceId;
 
     private String processHost;
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java
index 41ee1ef2ee..5500c744bd 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/StateEventResponseCommand.java
@@ -61,7 +61,7 @@ public class StateEventResponseCommand implements Serializable {
      */
     public Command convert2Command() {
         Command command = new Command();
-        command.setType(CommandType.TASK_EXECUTE_RESPONSE_ACK);
+        command.setType(CommandType.TASK_EXECUTE_RESULT_ACK);
         byte[] body = JSONUtils.toJsonByteArray(this);
         command.setBody(body);
         return command;
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java
similarity index 66%
copy from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java
copy to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java
index 000e3f4e02..f04c6107c1 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskDispatchCommand.java
@@ -20,24 +20,36 @@ package org.apache.dolphinscheduler.remote.command;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 
-import java.io.Serializable;
-
-import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
+import lombok.ToString;
+
 
+/**
+ * The task dispatch message, means dispatch a task to worker.
+ */
 @Data
 @NoArgsConstructor
-@AllArgsConstructor
-public class TaskExecuteRequestCommand implements Serializable {
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+public class TaskDispatchCommand extends BaseCommand {
 
     private static final long serialVersionUID = -1L;
 
     private TaskExecutionContext taskExecutionContext;
 
+    public TaskDispatchCommand(TaskExecutionContext taskExecutionContext,
+                               String messageSenderAddress,
+                               String messageReceiverAddress,
+                               long messageSendTime) {
+        super(messageSenderAddress, messageReceiverAddress, messageSendTime);
+        this.taskExecutionContext = taskExecutionContext;
+    }
+
     public Command convert2Command() {
         Command command = new Command();
-        command.setType(CommandType.TASK_EXECUTE_REQUEST);
+        command.setType(CommandType.TASK_DISPATCH_REQUEST);
         byte[] body = JSONUtils.toJsonByteArray(this);
         command.setBody(body);
         return command;
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java
similarity index 62%
copy from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseAckCommand.java
copy to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java
index f3df257b9d..a70cf8f239 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseAckCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteAckCommand.java
@@ -19,39 +19,31 @@ package org.apache.dolphinscheduler.remote.command;
 
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 
-import java.io.Serializable;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
 
 /**
  * task execute response ack command
  * from master to worker
  */
-public class TaskExecuteResponseAckCommand implements Serializable {
+@Data
+@NoArgsConstructor
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+public class TaskExecuteAckCommand extends BaseCommand {
 
     private int taskInstanceId;
     private int status;
 
-    public TaskExecuteResponseAckCommand() {
-        super();
-    }
-
-    public TaskExecuteResponseAckCommand(int status, int taskInstanceId) {
-        this.status = status;
-        this.taskInstanceId = taskInstanceId;
-    }
-
-    public int getStatus() {
-        return status;
-    }
-
-    public void setStatus(int status) {
+    public TaskExecuteAckCommand(int status,
+                                 int taskInstanceId,
+                                 String sourceServerAddress,
+                                 String messageReceiverAddress,
+                                 long messageSendTime) {
+        super(sourceServerAddress, messageReceiverAddress, messageSendTime);
         this.status = status;
-    }
-
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
         this.taskInstanceId = taskInstanceId;
     }
 
@@ -62,17 +54,9 @@ public class TaskExecuteResponseAckCommand implements Serializable {
      */
     public Command convert2Command() {
         Command command = new Command();
-        command.setType(CommandType.TASK_EXECUTE_RESPONSE_ACK);
+        command.setType(CommandType.TASK_EXECUTE_RESULT_ACK);
         byte[] body = JSONUtils.toJsonByteArray(this);
         command.setBody(body);
         return command;
     }
-
-    @Override
-    public String toString() {
-        return "TaskExecuteResponseAckCommand{"
-            + "taskInstanceId=" + taskInstanceId
-            + ", status=" + status
-            + '}';
-    }
 }
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java
deleted file mode 100644
index 4574b8818c..0000000000
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseCommand.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.remote.command;
-
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-
-import java.io.Serializable;
-import java.util.Date;
-
-/**
- * execute task response command
- */
-public class TaskExecuteResponseCommand implements Serializable {
-
-    public TaskExecuteResponseCommand() {
-    }
-
-    public TaskExecuteResponseCommand(int taskInstanceId, int processInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-        this.processInstanceId = processInstanceId;
-    }
-
-    /**
-     * task instance id
-     */
-    private int taskInstanceId;
-
-    /**
-     * process instance id
-     */
-    private int processInstanceId;
-
-    /**
-     * status
-     */
-    private int status;
-
-    /**
-     * startTime
-     */
-    private Date startTime;
-
-    /**
-     * host
-     */
-    private String host;
-
-    /**
-     * logPath
-     */
-    private String logPath;
-
-    /**
-     * executePath
-     */
-    private String executePath;
-
-
-    /**
-     * end time
-     */
-    private Date endTime;
-
-
-    /**
-     * processId
-     */
-    private int processId;
-
-    /**
-     * appIds
-     */
-    private String appIds;
-
-    /**
-     * varPool string
-     */
-    private String varPool;
-
-    public Date getStartTime() {
-        return startTime;
-    }
-
-    public void setStartTime(Date startTime) {
-        this.startTime = startTime;
-    }
-
-    public String getHost() {
-        return host;
-    }
-
-    public void setHost(String host) {
-        this.host = host;
-    }
-
-    public String getLogPath() {
-        return logPath;
-    }
-
-    public void setLogPath(String logPath) {
-        this.logPath = logPath;
-    }
-
-    public String getExecutePath() {
-        return executePath;
-    }
-
-    public void setExecutePath(String executePath) {
-        this.executePath = executePath;
-    }
-
-    public void setVarPool(String varPool) {
-        this.varPool = varPool;
-    }
-
-    public String getVarPool() {
-        return varPool;
-    }
-
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-    }
-
-    public int getStatus() {
-        return status;
-    }
-
-    public void setStatus(int status) {
-        this.status = status;
-    }
-
-    public Date getEndTime() {
-        return endTime;
-    }
-
-    public void setEndTime(Date endTime) {
-        this.endTime = endTime;
-    }
-
-    public int getProcessId() {
-        return processId;
-    }
-
-    public void setProcessId(int processId) {
-        this.processId = processId;
-    }
-
-    public String getAppIds() {
-        return appIds;
-    }
-
-    public void setAppIds(String appIds) {
-        this.appIds = appIds;
-    }
-
-    /**
-     * package response command
-     *
-     * @return command
-     */
-    public Command convert2Command() {
-        Command command = new Command();
-        command.setType(CommandType.TASK_EXECUTE_RESPONSE);
-        byte[] body = JSONUtils.toJsonByteArray(this);
-        command.setBody(body);
-        return command;
-    }
-
-    @Override
-    public String toString() {
-        return "TaskExecuteResponseCommand{"
-                + "taskInstanceId=" + taskInstanceId
-                + ", processInstanceId=" + processInstanceId
-                + ", status=" + status
-                + ", startTime=" + startTime
-                + ", endTime=" + endTime
-                + ", host=" + host
-                + ", logPath=" + logPath
-                + ", executePath=" + executePath
-                + ", processId=" + processId
-                + ", appIds='" + appIds + '\''
-                + ", varPool=" + varPool
-                + '}';
-    }
-
-    public int getProcessInstanceId() {
-        return processInstanceId;
-    }
-
-    public void setProcessInstanceId(int processInstanceId) {
-        this.processInstanceId = processInstanceId;
-    }
-}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java
similarity index 51%
rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseAckCommand.java
rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java
index f3df257b9d..12dfa56976 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResponseAckCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteResultCommand.java
@@ -19,41 +19,82 @@ package org.apache.dolphinscheduler.remote.command;
 
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 
-import java.io.Serializable;
+import java.util.Date;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
 
 /**
- * task execute response ack command
- * from master to worker
+ * execute task response command
  */
-public class TaskExecuteResponseAckCommand implements Serializable {
+@Data
+@NoArgsConstructor
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+public class TaskExecuteResultCommand extends BaseCommand {
+
+    public TaskExecuteResultCommand(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) {
+        super(messageSenderAddress, messageReceiverAddress, messageSendTime);
+    }
 
+    /**
+     * task instance id
+     */
     private int taskInstanceId;
+
+    /**
+     * process instance id
+     */
+    private int processInstanceId;
+
+    /**
+     * status
+     */
     private int status;
 
-    public TaskExecuteResponseAckCommand() {
-        super();
-    }
+    /**
+     * startTime
+     */
+    private Date startTime;
 
-    public TaskExecuteResponseAckCommand(int status, int taskInstanceId) {
-        this.status = status;
-        this.taskInstanceId = taskInstanceId;
-    }
+    /**
+     * host
+     */
+    private String host;
 
-    public int getStatus() {
-        return status;
-    }
+    /**
+     * logPath
+     */
+    private String logPath;
 
-    public void setStatus(int status) {
-        this.status = status;
-    }
+    /**
+     * executePath
+     */
+    private String executePath;
 
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
 
-    public void setTaskInstanceId(int taskInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-    }
+    /**
+     * end time
+     */
+    private Date endTime;
+
+
+    /**
+     * processId
+     */
+    private int processId;
+
+    /**
+     * appIds
+     */
+    private String appIds;
+
+    /**
+     * varPool string
+     */
+    private String varPool;
 
     /**
      * package response command
@@ -62,17 +103,9 @@ public class TaskExecuteResponseAckCommand implements Serializable {
      */
     public Command convert2Command() {
         Command command = new Command();
-        command.setType(CommandType.TASK_EXECUTE_RESPONSE_ACK);
+        command.setType(CommandType.TASK_EXECUTE_RESULT);
         byte[] body = JSONUtils.toJsonByteArray(this);
         command.setBody(body);
         return command;
     }
-
-    @Override
-    public String toString() {
-        return "TaskExecuteResponseAckCommand{"
-            + "taskInstanceId=" + taskInstanceId
-            + ", status=" + status
-            + '}';
-    }
 }
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java
similarity index 92%
rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckCommand.java
rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java
index b0bb666cba..3e17ade0ad 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningAckMessage.java
@@ -25,16 +25,16 @@ import java.io.Serializable;
  * task execute running ack command
  * from master to worker
  */
-public class TaskExecuteRunningAckCommand implements Serializable {
+public class TaskExecuteRunningAckMessage implements Serializable {
 
     private int taskInstanceId;
     private int status;
 
-    public TaskExecuteRunningAckCommand() {
+    public TaskExecuteRunningAckMessage() {
         super();
     }
 
-    public TaskExecuteRunningAckCommand(int status, int taskInstanceId) {
+    public TaskExecuteRunningAckMessage(int status, int taskInstanceId) {
         this.status = status;
         this.taskInstanceId = taskInstanceId;
     }
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java
index 0a2eac29f3..241d37744a 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRunningCommand.java
@@ -19,14 +19,21 @@ package org.apache.dolphinscheduler.remote.command;
 
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 
-import java.io.Serializable;
 import java.util.Date;
 
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
 /**
- * task execute running command
- * from worker to master
+ * Task running message, means the task is running in worker.
  */
-public class TaskExecuteRunningCommand implements Serializable {
+@Data
+@NoArgsConstructor
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+public class TaskExecuteRunningCommand extends BaseCommand {
 
     /**
      * taskInstanceId
@@ -73,76 +80,8 @@ public class TaskExecuteRunningCommand implements Serializable {
      */
     private String appIds;
 
-    public Date getStartTime() {
-        return startTime;
-    }
-
-    public void setStartTime(Date startTime) {
-        this.startTime = startTime;
-    }
-
-    public String getHost() {
-        return host;
-    }
-
-    public void setHost(String host) {
-        this.host = host;
-    }
-
-    public int getStatus() {
-        return status;
-    }
-
-    public void setStatus(int status) {
-        this.status = status;
-    }
-
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-    }
-
-    public int getProcessInstanceId() {
-        return processInstanceId;
-    }
-
-    public void setProcessInstanceId(int processInstanceId) {
-        this.processInstanceId = processInstanceId;
-    }
-
-    public String getLogPath() {
-        return logPath;
-    }
-
-    public void setLogPath(String logPath) {
-        this.logPath = logPath;
-    }
-
-    public String getExecutePath() {
-        return executePath;
-    }
-
-    public void setExecutePath(String executePath) {
-        this.executePath = executePath;
-    }
-
-    public int getProcessId() {
-        return processId;
-    }
-
-    public void setProcessId(int processId) {
-        this.processId = processId;
-    }
-
-    public String getAppIds() {
-        return appIds;
-    }
-
-    public void setAppIds(String appIds) {
-        this.appIds = appIds;
+    public TaskExecuteRunningCommand(String messageSenderAddress, String messageReceiverAddress, long messageSendTime) {
+        super(messageSenderAddress, messageReceiverAddress, messageSendTime);
     }
 
     /**
@@ -158,18 +97,4 @@ public class TaskExecuteRunningCommand implements Serializable {
         return command;
     }
 
-    @Override
-    public String toString() {
-        return "TaskExecuteRunningCommand{"
-                + "taskInstanceId=" + taskInstanceId
-                + ", processInstanceId='" + processInstanceId + '\''
-                + ", startTime=" + startTime
-                + ", host='" + host + '\''
-                + ", status=" + status
-                + ", logPath='" + logPath + '\''
-                + ", executePath='" + executePath + '\''
-                + ", processId=" + processId + '\''
-                + ", appIds='" + appIds + '\''
-                + '}';
-    }
 }
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRejectAckCommand.java
similarity index 61%
copy from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java
copy to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRejectAckCommand.java
index 4fc752e4b0..aed647bb4c 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/HostUpdateCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRejectAckCommand.java
@@ -19,54 +19,41 @@ package org.apache.dolphinscheduler.remote.command;
 
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 
-import java.io.Serializable;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
 
-/**
- * process host update
- */
-public class HostUpdateCommand implements Serializable {
+@Data
+@NoArgsConstructor
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+public class TaskRejectAckCommand extends BaseCommand {
 
-    /**
-     * task id
-     */
     private int taskInstanceId;
-
-    private String processHost;
-
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
+    private int status;
+
+    public TaskRejectAckCommand(int status,
+                                int taskInstanceId,
+                                String messageSenderAddress,
+                                String messageReceiverAddress,
+                                long messageSendTime) {
+        super(messageSenderAddress, messageReceiverAddress, messageSendTime);
+        this.status = status;
         this.taskInstanceId = taskInstanceId;
     }
 
-    public String getProcessHost() {
-        return processHost;
-    }
-
-    public void setProcessHost(String processHost) {
-        this.processHost = processHost;
-    }
-
     /**
-     * package request command
+     * package response command
      *
      * @return command
      */
     public Command convert2Command() {
         Command command = new Command();
-        command.setType(CommandType.PROCESS_HOST_UPDATE_REQUEST);
+        command.setType(CommandType.TASK_REJECT_ACK);
         byte[] body = JSONUtils.toJsonByteArray(this);
         command.setBody(body);
         return command;
     }
 
-    @Override
-    public String toString() {
-        return "HostUpdateCommand{"
-                + "taskInstanceId=" + taskInstanceId
-                + "host=" + processHost
-                + '}';
-    }
 }
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRecallCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRejectCommand.java
similarity index 100%
copy from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRecallCommand.java
copy to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRejectCommand.java
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java
index dc8e1f0d36..edbaea13ce 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java
@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.remote.utils;
 
+import lombok.NonNull;
 import static org.apache.dolphinscheduler.common.Constants.COLON;
 
 import java.io.Serializable;
@@ -95,7 +96,7 @@ public class Host implements Serializable {
      * @param address address
      * @return host
      */
-    public static Host of(String address) {
+    public static Host of(@NonNull String address) {
         String[] parts = splitAddress(address);
         return new Host(parts[0], Integer.parseInt(parts[1]));
     }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
index 93dc6309db..0321b76f44 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
@@ -25,9 +25,10 @@ import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.plugin.task.api.ProcessUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
-import org.apache.dolphinscheduler.server.worker.prc.WorkerRpcServer;
+import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcServer;
 import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient;
-import org.apache.dolphinscheduler.server.worker.runner.RetryReportTaskStatusThread;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
@@ -71,9 +72,6 @@ public class WorkerServer implements IStoppable {
     @Autowired
     private AlertClientService alertClientService;
 
-    @Autowired
-    private RetryReportTaskStatusThread retryReportTaskStatusThread;
-
     @Autowired
     private WorkerManagerThread workerManagerThread;
 
@@ -89,6 +87,12 @@ public class WorkerServer implements IStoppable {
     @Autowired
     private WorkerRpcServer workerRpcServer;
 
+    @Autowired
+    private WorkerRpcClient workerRpcClient;
+
+    @Autowired
+    private MessageRetryRunner messageRetryRunner;
+
     /**
      * worker server startup, not use web service
      *
@@ -102,7 +106,7 @@ public class WorkerServer implements IStoppable {
     @PostConstruct
     public void run() {
         this.workerRpcServer.start();
-
+        this.workerRpcClient.start();
         this.taskPluginManager.installPlugin();
 
         this.workerRegistryClient.registry();
@@ -112,7 +116,7 @@ public class WorkerServer implements IStoppable {
 
         this.workerManagerThread.start();
 
-        this.retryReportTaskStatusThread.start();
+        this.messageRetryRunner.start();
 
         /*
          * registry hooks, which are called before the process exits
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java
index a261fec0a1..e69de29bb2 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.worker.cache;
-
-import org.apache.dolphinscheduler.common.enums.TaskEventType;
-import org.apache.dolphinscheduler.remote.command.Command;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * Response Cache : cache worker send master result
- */
-public class ResponseCache {
-
-    private static final ResponseCache instance = new ResponseCache();
-
-    private ResponseCache() {
-    }
-
-    public static ResponseCache get() {
-        return instance;
-    }
-
-    private Map<Integer, Command> runningCache = new ConcurrentHashMap<>();
-    private Map<Integer, Command> responseCache = new ConcurrentHashMap<>();
-
-    /**
-     * cache response
-     *
-     * @param taskInstanceId taskInstanceId
-     * @param command command
-     * @param event event ACK/RESULT
-     */
-    public void cache(Integer taskInstanceId, Command command, TaskEventType event) {
-        switch (event) {
-            case RUNNING:
-                runningCache.put(taskInstanceId, command);
-                break;
-            case RESULT:
-                responseCache.put(taskInstanceId, command);
-                break;
-            default:
-                throw new IllegalArgumentException("invalid event type : " + event);
-        }
-    }
-
-    /**
-     * remove running cache
-     *
-     * @param taskInstanceId taskInstanceId
-     */
-    public void removeRunningCache(Integer taskInstanceId) {
-        runningCache.remove(taskInstanceId);
-    }
-
-    /**
-     * remove response cache
-     *
-     * @param taskInstanceId taskInstanceId
-     */
-    public void removeResponseCache(Integer taskInstanceId) {
-        responseCache.remove(taskInstanceId);
-    }
-
-    /**
-     * get running cache
-     *
-     * @return getAckCache
-     */
-    public Map<Integer, Command> getRunningCache() {
-        return runningCache;
-    }
-
-    /**
-     * getResponseCache
-     *
-     * @return getResponseCache
-     */
-    public Map<Integer, Command> getResponseCache() {
-        return responseCache;
-    }
-}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
index a641b28e1b..aab162d6d4 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
@@ -17,6 +17,8 @@
 
 package org.apache.dolphinscheduler.server.worker.config;
 
+import org.apache.dolphinscheduler.common.utils.NetUtils;
+
 import java.time.Duration;
 import java.util.Set;
 
@@ -46,6 +48,10 @@ public class WorkerConfig implements Validator {
     private Set<String> groups = Sets.newHashSet("default");
     private String alertListenHost = "localhost";
     private int alertListenPort = 50052;
+    /**
+     * This field doesn't need to set at config file, it will be calculated by workerIp:listenPort
+     */
+    private String workerAddress;
 
     @Override
     public boolean supports(Class<?> clazz) {
@@ -64,6 +70,6 @@ public class WorkerConfig implements Validator {
         if (workerConfig.getMaxCpuLoadAvg() <= 0) {
             workerConfig.setMaxCpuLoadAvg(Runtime.getRuntime().availableProcessors() * 2);
         }
-
+        workerConfig.setWorkerAddress(NetUtils.getAddr(workerConfig.getListenPort()));
     }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java
new file mode 100644
index 0000000000..18dceb069b
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageRetryRunner.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.message;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
+import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.remote.command.BaseCommand;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.annotation.PostConstruct;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.stereotype.Component;
+
+import lombok.NonNull;
+
+@Component
+public class MessageRetryRunner extends BaseDaemonThread {
+
+    private final Logger logger = LoggerFactory.getLogger(MessageRetryRunner.class);
+
+    protected MessageRetryRunner() {
+        super("WorkerMessageRetryRunnerThread");
+    }
+
+    private static long MESSAGE_RETRY_WINDOW = Duration.ofMinutes(5L).toMillis();
+
+    @Autowired
+    private ApplicationContext applicationContext;
+
+    private Map<CommandType, MessageSender<BaseCommand>> messageSenderMap = new HashMap<>();
+
+    private Map<Integer, Map<CommandType, BaseCommand>> needToRetryMessages = new ConcurrentHashMap<>();
+
+    @PostConstruct
+    public void init() {
+        Map<String, MessageSender> messageSenders = applicationContext.getBeansOfType(MessageSender.class);
+        messageSenders.values().forEach(messageSender -> {
+            messageSenderMap.put(messageSender.getMessageType(), messageSender);
+            logger.info("Injected message sender: {}", messageSender.getClass().getName());
+        });
+    }
+
+    @Override
+    public synchronized void start() {
+        logger.info("Message retry runner staring");
+        super.start();
+        logger.info("Message retry runner started");
+    }
+
+    public void addRetryMessage(int taskInstanceId, @NonNull CommandType messageType, BaseCommand baseCommand) {
+        needToRetryMessages.computeIfAbsent(taskInstanceId, k -> new ConcurrentHashMap<>()).put(messageType,
+                                                                                                baseCommand);
+    }
+
+    public void removeRetryMessage(int taskInstanceId, @NonNull CommandType messageType) {
+        Map<CommandType, BaseCommand> retryMessages = needToRetryMessages.get(taskInstanceId);
+        if (retryMessages != null) {
+            retryMessages.remove(messageType);
+        }
+    }
+
+    public void removeRetryMessages(int taskInstanceId) {
+        needToRetryMessages.remove(taskInstanceId);
+    }
+
+    public void updateMessageHost(int taskInstanceId, String messageReceiverHost) {
+        Map<CommandType, BaseCommand> needToRetryMessages = this.needToRetryMessages.get(taskInstanceId);
+        if (needToRetryMessages != null) {
+            needToRetryMessages.values().forEach(baseMessage -> {
+                baseMessage.setMessageReceiverAddress(messageReceiverHost);
+            });
+        }
+    }
+
+    public void run() {
+        while (Stopper.isRunning()) {
+            try {
+                if (needToRetryMessages.isEmpty()) {
+                    Thread.sleep(MESSAGE_RETRY_WINDOW);
+                }
+
+                long now = System.currentTimeMillis();
+                for (Map.Entry<Integer, Map<CommandType, BaseCommand>> taskEntry : needToRetryMessages.entrySet()) {
+                    Integer taskInstanceId = taskEntry.getKey();
+                    LoggerUtils.setTaskInstanceIdMDC(taskInstanceId);
+                    try {
+                        for (Map.Entry<CommandType, BaseCommand> messageEntry : taskEntry.getValue().entrySet()) {
+                            CommandType messageType = messageEntry.getKey();
+                            BaseCommand message = messageEntry.getValue();
+                            if (now - message.getMessageSendTime() > MESSAGE_RETRY_WINDOW) {
+                                logger.info("Begin retry send message to master, message: {}", message);
+                                message.setMessageSendTime(now);
+                                messageSenderMap.get(messageType).sendMessage(message);
+                                logger.info("Success send message to master, message: {}", message);
+                            }
+                        }
+                    } catch (Exception e) {
+                        logger.warn("Retry send message to master error", e);
+                    } finally {
+                        LoggerUtils.removeTaskInstanceIdMDC();
+                    }
+                }
+                Thread.sleep(Constants.SLEEP_TIME_MILLIS);
+            } catch (InterruptedException instance) {
+                logger.warn("The message retry thread is interrupted, will break this loop", instance);
+                Thread.currentThread().interrupt();
+                break;
+            } catch (Exception ex) {
+                logger.error("Retry send message failed, get an known exception.", ex);
+            }
+        }
+    }
+}
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageSender.java
similarity index 53%
rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageSender.java
index 000e3f4e02..519c5b2b3e 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/MessageSender.java
@@ -15,32 +15,29 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.remote.command;
+package org.apache.dolphinscheduler.server.worker.message;
 
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-
-import java.io.Serializable;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskExecuteRequestCommand implements Serializable {
-
-    private static final long serialVersionUID = -1L;
-
-    private TaskExecutionContext taskExecutionContext;
-
-    public Command convert2Command() {
-        Command command = new Command();
-        command.setType(CommandType.TASK_EXECUTE_REQUEST);
-        byte[] body = JSONUtils.toJsonByteArray(this);
-        command.setBody(body);
-        return command;
-    }
-
+import org.apache.dolphinscheduler.remote.command.BaseCommand;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
+
+public interface MessageSender<T extends BaseCommand> {
+
+    /**
+     * Send the message
+     *
+     * @throws RemotingException Cannot connect to the target host.
+     */
+    void sendMessage(T message) throws RemotingException;
+
+    /**
+     * Build the message from task context and message received address.
+     */
+    T buildMessage(TaskExecutionContext taskExecutionContext, String messageReceiverAddress);
+
+    /**
+     * The message type can be sent by this sender.
+     */
+    CommandType getMessageType();
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java
new file mode 100644
index 0000000000..a7d05a4791
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteResultMessageSender.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.message;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteResultCommand;
+import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
+import org.apache.dolphinscheduler.remote.utils.Host;
+import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TaskExecuteResultMessageSender implements MessageSender<TaskExecuteResultCommand> {
+
+    @Autowired
+    private WorkerConfig workerConfig;
+
+    @Autowired
+    private WorkerRpcClient workerRpcClient;
+
+    @Override
+    public void sendMessage(TaskExecuteResultCommand message) throws RemotingException {
+        workerRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command());
+    }
+
+    public TaskExecuteResultCommand buildMessage(TaskExecutionContext taskExecutionContext,
+                                                 String messageReceiverAddress) {
+        TaskExecuteResultCommand taskExecuteResultMessage
+            = new TaskExecuteResultCommand(workerConfig.getWorkerAddress(),
+                                           messageReceiverAddress,
+                                           System.currentTimeMillis());
+        taskExecuteResultMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId());
+        taskExecuteResultMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+        taskExecuteResultMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode());
+        taskExecuteResultMessage.setLogPath(taskExecutionContext.getLogPath());
+        taskExecuteResultMessage.setExecutePath(taskExecutionContext.getExecutePath());
+        taskExecuteResultMessage.setAppIds(taskExecutionContext.getAppIds());
+        taskExecuteResultMessage.setProcessId(taskExecutionContext.getProcessId());
+        taskExecuteResultMessage.setHost(taskExecutionContext.getHost());
+        taskExecuteResultMessage.setStartTime(taskExecutionContext.getStartTime());
+        taskExecuteResultMessage.setEndTime(taskExecutionContext.getEndTime());
+        taskExecuteResultMessage.setVarPool(taskExecutionContext.getVarPool());
+        taskExecuteResultMessage.setExecutePath(taskExecutionContext.getExecutePath());
+        return taskExecuteResultMessage;
+    }
+
+    @Override
+    public CommandType getMessageType() {
+        return CommandType.TASK_EXECUTE_RESULT;
+    }
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteRunningMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteRunningMessageSender.java
new file mode 100644
index 0000000000..7891f4be70
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskExecuteRunningMessageSender.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.message;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningCommand;
+import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
+import org.apache.dolphinscheduler.remote.utils.Host;
+import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import lombok.NonNull;
+
+@Component
+public class TaskExecuteRunningMessageSender implements MessageSender<TaskExecuteRunningCommand> {
+
+    @Autowired
+    private WorkerRpcClient workerRpcClient;
+
+    @Autowired
+    private WorkerConfig workerConfig;
+
+    @Override
+    public void sendMessage(TaskExecuteRunningCommand message) throws RemotingException {
+        workerRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command());
+    }
+
+    public TaskExecuteRunningCommand buildMessage(@NonNull TaskExecutionContext taskExecutionContext,
+                                                  @NonNull String messageReceiverAddress) {
+        TaskExecuteRunningCommand taskExecuteRunningMessage
+            = new TaskExecuteRunningCommand(workerConfig.getWorkerAddress(),
+                                            messageReceiverAddress,
+                                            System.currentTimeMillis());
+        taskExecuteRunningMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+        taskExecuteRunningMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId());
+        taskExecuteRunningMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode());
+        taskExecuteRunningMessage.setLogPath(taskExecutionContext.getLogPath());
+        taskExecuteRunningMessage.setHost(taskExecutionContext.getHost());
+        taskExecuteRunningMessage.setStartTime(taskExecutionContext.getStartTime());
+        taskExecuteRunningMessage.setExecutePath(taskExecutionContext.getExecutePath());
+        return taskExecuteRunningMessage;
+    }
+
+    @Override
+    public CommandType getMessageType() {
+        return CommandType.TASK_EXECUTE_RUNNING;
+    }
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java
new file mode 100644
index 0000000000..d50c5d8997
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/message/TaskRejectMessageSender.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.message;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.command.TaskRejectCommand;
+import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
+import org.apache.dolphinscheduler.remote.utils.Host;
+import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcClient;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TaskRejectMessageSender implements MessageSender<TaskRejectCommand> {
+
+    @Autowired
+    private WorkerRpcClient workerRpcClient;
+
+    @Autowired
+    private WorkerConfig workerConfig;
+
+    @Override
+    public void sendMessage(TaskRejectCommand message) throws RemotingException {
+        workerRpcClient.send(Host.of(message.getMessageReceiverAddress()), message.convert2Command());
+    }
+
+    public TaskRejectCommand buildMessage(TaskExecutionContext taskExecutionContext, String masterAddress) {
+        TaskRejectCommand taskRejectMessage = new TaskRejectCommand(workerConfig.getWorkerAddress(),
+                                                                    masterAddress,
+                                                                    System.currentTimeMillis());
+        taskRejectMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+        taskRejectMessage.setProcessInstanceId(taskExecutionContext.getProcessInstanceId());
+        taskRejectMessage.setHost(taskExecutionContext.getHost());
+        return taskRejectMessage;
+    }
+
+    @Override
+    public CommandType getMessageType() {
+        return CommandType.TASK_REJECT;
+    }
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java
index d1ceca6c9e..d36c80012d 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/HostUpdateProcessor.java
@@ -21,8 +21,8 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.command.HostUpdateCommand;
-import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
+import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -42,22 +42,19 @@ public class HostUpdateProcessor implements NettyRequestProcessor {
 
     private final Logger logger = LoggerFactory.getLogger(HostUpdateProcessor.class);
 
-    /**
-     * task callback service
-     */
     @Autowired
-    private TaskCallbackService taskCallbackService;
+    private MessageRetryRunner messageRetryRunner;
 
     @Override
     public void process(Channel channel, Command command) {
-        Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType()));
+        Preconditions.checkArgument(CommandType.PROCESS_HOST_UPDATE_REQUEST == command.getType(),
+                                    String.format("invalid command type : %s", command.getType()));
         HostUpdateCommand updateCommand = JSONUtils.parseObject(command.getBody(), HostUpdateCommand.class);
         if (updateCommand == null) {
             logger.error("host update command is null");
             return;
         }
         logger.info("received host update command : {}", updateCommand);
-        taskCallbackService.changeRemoteChannel(updateCommand.getTaskInstanceId(), new NettyRemoteChannel(channel, command.getOpaque()));
-
+        messageRetryRunner.updateMessageHost(updateCommand.getTaskInstanceId(), updateCommand.getProcessHost());
     }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
index 626cc6f7e5..e69de29bb2 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.worker.processor;
-
-import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS;
-
-import org.apache.dolphinscheduler.common.enums.TaskEventType;
-import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.remote.NettyRemotingClient;
-import org.apache.dolphinscheduler.remote.command.Command;
-import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningCommand;
-import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand;
-import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
-import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel;
-import org.apache.dolphinscheduler.server.worker.cache.ResponseCache;
-
-import java.util.Arrays;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-
-
-/**
- * task callback service
- */
-@Service
-public class TaskCallbackService {
-
-    private final Logger logger = LoggerFactory.getLogger(TaskCallbackService.class);
-    private static final int[] RETRY_BACKOFF = {1, 2, 3, 5, 10, 20, 40, 100, 100, 100, 100, 200, 200, 200};
-
-    @Autowired
-    private TaskExecuteRunningAckProcessor taskExecuteRunningProcessor;
-
-    @Autowired
-    private TaskExecuteResponseAckProcessor taskExecuteResponseAckProcessor;
-
-    /**
-     * remote channels
-     */
-    private static final ConcurrentHashMap<Integer, NettyRemoteChannel> REMOTE_CHANNELS = new ConcurrentHashMap<>();
-
-    /**
-     * netty remoting client
-     */
-    private final NettyRemotingClient nettyRemotingClient;
-
-    public TaskCallbackService() {
-        final NettyClientConfig clientConfig = new NettyClientConfig();
-        this.nettyRemotingClient = new NettyRemotingClient(clientConfig);
-        this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RUNNING_ACK, taskExecuteRunningProcessor);
-        this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE_ACK, taskExecuteResponseAckProcessor);
-    }
-
-    /**
-     * add callback channel
-     *
-     * @param taskInstanceId taskInstanceId
-     * @param channel channel
-     */
-    public void addRemoteChannel(int taskInstanceId, NettyRemoteChannel channel) {
-        REMOTE_CHANNELS.put(taskInstanceId, channel);
-    }
-
-    /**
-     * change remote channel
-     */
-    public void changeRemoteChannel(int taskInstanceId, NettyRemoteChannel channel) {
-        REMOTE_CHANNELS.put(taskInstanceId, channel);
-    }
-
-    /**
-     * get callback channel
-     *
-     * @param taskInstanceId taskInstanceId
-     * @return callback channel
-     */
-    private Optional<NettyRemoteChannel> getRemoteChannel(int taskInstanceId) {
-        Channel newChannel;
-        NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(taskInstanceId);
-        if (nettyRemoteChannel != null) {
-            if (nettyRemoteChannel.isActive()) {
-                return Optional.of(nettyRemoteChannel);
-            }
-            newChannel = nettyRemotingClient.getChannel(nettyRemoteChannel.getHost());
-            if (newChannel != null) {
-                return Optional.of(getRemoteChannel(newChannel, nettyRemoteChannel.getOpaque(), taskInstanceId));
-            }
-        }
-        return Optional.empty();
-    }
-
-    public long pause(int ntries) {
-        return SLEEP_TIME_MILLIS * RETRY_BACKOFF[ntries % RETRY_BACKOFF.length];
-    }
-
-    private NettyRemoteChannel getRemoteChannel(Channel newChannel, long opaque, int taskInstanceId) {
-        NettyRemoteChannel remoteChannel = new NettyRemoteChannel(newChannel, opaque);
-        addRemoteChannel(taskInstanceId, remoteChannel);
-        return remoteChannel;
-    }
-
-    private NettyRemoteChannel getRemoteChannel(Channel newChannel, int taskInstanceId) {
-        NettyRemoteChannel remoteChannel = new NettyRemoteChannel(newChannel);
-        addRemoteChannel(taskInstanceId, remoteChannel);
-        return remoteChannel;
-    }
-
-    /**
-     * remove callback channels
-     *
-     * @param taskInstanceId taskInstanceId
-     */
-    public static void remove(int taskInstanceId) {
-        REMOTE_CHANNELS.remove(taskInstanceId);
-    }
-
-    /**
-     * send result
-     *
-     * @param taskInstanceId taskInstanceId
-     * @param command command
-     */
-    public void send(int taskInstanceId, Command command) {
-        Optional<NettyRemoteChannel> nettyRemoteChannel = getRemoteChannel(taskInstanceId);
-        if (nettyRemoteChannel.isPresent()) {
-            nettyRemoteChannel.get().writeAndFlush(command).addListener(new ChannelFutureListener() {
-                @Override
-                public void operationComplete(ChannelFuture future) {
-                    if (!future.isSuccess()) {
-                        logger.error("Send callback command error, taskInstanceId: {}, command: {}", taskInstanceId, command);
-                    }
-                }
-            });
-        } else {
-            logger.warn("Remote channel of taskInstanceId is null: {}, cannot send command: {}", taskInstanceId, command);
-        }
-    }
-
-    /**
-     * build task execute running command
-     *
-     * @param taskExecutionContext taskExecutionContext
-     * @return TaskExecuteAckCommand
-     */
-    private TaskExecuteRunningCommand buildTaskExecuteRunningCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteRunningCommand command = new TaskExecuteRunningCommand();
-        command.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-        command.setProcessInstanceId(taskExecutionContext.getProcessInstanceId());
-        command.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode());
-        command.setLogPath(taskExecutionContext.getLogPath());
-        command.setHost(taskExecutionContext.getHost());
-        command.setStartTime(taskExecutionContext.getStartTime());
-        command.setExecutePath(taskExecutionContext.getExecutePath());
-        return command;
-    }
-
-    /**
-     * build task execute response command
-     *
-     * @param taskExecutionContext taskExecutionContext
-     * @return TaskExecuteResponseCommand
-     */
-    private TaskExecuteResponseCommand buildTaskExecuteResponseCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteResponseCommand command = new TaskExecuteResponseCommand();
-        command.setProcessInstanceId(taskExecutionContext.getProcessInstanceId());
-        command.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-        command.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode());
-        command.setLogPath(taskExecutionContext.getLogPath());
-        command.setExecutePath(taskExecutionContext.getExecutePath());
-        command.setAppIds(taskExecutionContext.getAppIds());
-        command.setProcessId(taskExecutionContext.getProcessId());
-        command.setHost(taskExecutionContext.getHost());
-        command.setStartTime(taskExecutionContext.getStartTime());
-        command.setEndTime(taskExecutionContext.getEndTime());
-        command.setVarPool(taskExecutionContext.getVarPool());
-        command.setExecutePath(taskExecutionContext.getExecutePath());
-        return command;
-    }
-
-    /**
-     * build TaskKillResponseCommand
-     *
-     * @param taskExecutionContext taskExecutionContext
-     * @return build TaskKillResponseCommand
-     */
-    private TaskKillResponseCommand buildKillTaskResponseCommand(TaskExecutionContext taskExecutionContext) {
-        TaskKillResponseCommand taskKillResponseCommand = new TaskKillResponseCommand();
-        taskKillResponseCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode());
-        taskKillResponseCommand.setAppIds(Arrays.asList(taskExecutionContext.getAppIds().split(TaskConstants.COMMA)));
-        taskKillResponseCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-        taskKillResponseCommand.setHost(taskExecutionContext.getHost());
-        taskKillResponseCommand.setProcessId(taskExecutionContext.getProcessId());
-        return taskKillResponseCommand;
-    }
-
-    /**
-     * send task execute running command
-     * todo unified callback command
-     */
-    public void sendTaskExecuteRunningCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteRunningCommand command = buildTaskExecuteRunningCommand(taskExecutionContext);
-        // add response cache
-        ResponseCache.get().cache(taskExecutionContext.getTaskInstanceId(), command.convert2Command(), TaskEventType.RUNNING);
-        send(taskExecutionContext.getTaskInstanceId(), command.convert2Command());
-    }
-
-    /**
-     * send task execute delay command
-     * todo unified callback command
-     */
-    public void sendTaskExecuteDelayCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteRunningCommand command = buildTaskExecuteRunningCommand(taskExecutionContext);
-        send(taskExecutionContext.getTaskInstanceId(), command.convert2Command());
-    }
-
-    /**
-     * send task execute response command
-     * todo unified callback command
-     */
-    public void sendTaskExecuteResponseCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteResponseCommand command = buildTaskExecuteResponseCommand(taskExecutionContext);
-        // add response cache
-        ResponseCache.get().cache(taskExecutionContext.getTaskInstanceId(), command.convert2Command(), TaskEventType.RESULT);
-        send(taskExecutionContext.getTaskInstanceId(), command.convert2Command());
-    }
-
-    public void sendTaskKillResponseCommand(TaskExecutionContext taskExecutionContext) {
-        TaskKillResponseCommand taskKillResponseCommand = buildKillTaskResponseCommand(taskExecutionContext);
-        send(taskExecutionContext.getTaskInstanceId(), taskKillResponseCommand.convert2Command());
-    }
-}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java
similarity index 79%
rename from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java
index 45ebbc2439..786b613621 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessor.java
@@ -24,19 +24,18 @@ import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
-import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
-import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel;
+import org.apache.dolphinscheduler.remote.command.TaskDispatchCommand;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.server.utils.LogUtils;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
 import org.apache.dolphinscheduler.server.worker.metrics.TaskMetrics;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender;
 import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
@@ -56,16 +55,13 @@ import io.micrometer.core.annotation.Timed;
 import io.netty.channel.Channel;
 
 /**
- * worker request processor
+ * Used to handle {@link CommandType#TASK_DISPATCH_REQUEST}
  */
 @Component
-public class TaskExecuteProcessor implements NettyRequestProcessor {
+public class TaskDispatchProcessor implements NettyRequestProcessor {
 
-    private static final Logger logger = LoggerFactory.getLogger(TaskExecuteProcessor.class);
+    private static final Logger logger = LoggerFactory.getLogger(TaskDispatchProcessor.class);
 
-    /**
-     * worker config
-     */
     @Autowired
     private WorkerConfig workerConfig;
 
@@ -73,7 +69,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
      * task callback service
      */
     @Autowired
-    private TaskCallbackService taskCallbackService;
+    private WorkerMessageSender workerMessageSender;
 
     /**
      * alert client service
@@ -97,19 +93,19 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
     @Timed(value = "ds.task.execution.duration", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
     @Override
     public void process(Channel channel, Command command) {
-        Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(),
+        Preconditions.checkArgument(CommandType.TASK_DISPATCH_REQUEST == command.getType(),
                                     String.format("invalid command type : %s", command.getType()));
 
-        TaskExecuteRequestCommand taskRequestCommand = JSONUtils.parseObject(command.getBody(),
-                                                                             TaskExecuteRequestCommand.class);
+        TaskDispatchCommand taskDispatchCommand = JSONUtils.parseObject(command.getBody(), TaskDispatchCommand.class);
 
-        if (taskRequestCommand == null) {
-            logger.error("task execute request command is null");
+        if (taskDispatchCommand == null) {
+            logger.error("task execute request command content is null");
             return;
         }
-        logger.info("task execute request command : {}", taskRequestCommand);
+        final String masterAddress = taskDispatchCommand.getMessageSenderAddress();
+        logger.info("task execute request message: {}", taskDispatchCommand);
 
-        TaskExecutionContext taskExecutionContext = taskRequestCommand.getTaskExecutionContext();
+        TaskExecutionContext taskExecutionContext = taskDispatchCommand.getTaskExecutionContext();
 
         if (taskExecutionContext == null) {
             logger.error("task execution context is null");
@@ -126,7 +122,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
 
             // todo custom logger
 
-            taskExecutionContext.setHost(NetUtils.getAddr(workerConfig.getListenPort()));
+            taskExecutionContext.setHost(workerConfig.getWorkerAddress());
             taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext));
 
             if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) {
@@ -142,7 +138,9 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
                     TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
                     taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
                     taskExecutionContext.setEndTime(new Date());
-                    taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                    workerMessageSender.sendMessageWithRetry(taskExecutionContext,
+                                                             masterAddress,
+                                                             CommandType.TASK_EXECUTE_RESULT);
                     return;
                 }
 
@@ -156,18 +154,17 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
                 } catch (Throwable ex) {
                     logger.error("create execLocalPath fail, path: {}, taskInstanceId: {}",
                                  execLocalPath,
-                                 taskExecutionContext.getTaskInstanceId());
-                    logger.error("create executeLocalPath fail", ex);
+                                 taskExecutionContext.getTaskInstanceId(),
+                                 ex);
                     TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
                     taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
-                    taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                    workerMessageSender.sendMessageWithRetry(taskExecutionContext,
+                                                             masterAddress,
+                                                             CommandType.TASK_EXECUTE_RESULT);
                     return;
                 }
             }
 
-            taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
-                                                 new NettyRemoteChannel(channel, command.getOpaque()));
-
             // delay task process
             long remainTime = DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(),
                                                       taskExecutionContext.getDelayTime() * 60L);
@@ -177,21 +174,22 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
                             remainTime);
                 taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION);
                 taskExecutionContext.setStartTime(null);
-                taskCallbackService.sendTaskExecuteDelayCommand(taskExecutionContext);
+                workerMessageSender.sendMessage(taskExecutionContext, masterAddress, CommandType.TASK_EXECUTE_RESULT);
             }
 
             // submit task to manager
             boolean offer = workerManager.offer(new TaskExecuteThread(taskExecutionContext,
-                                                                      taskCallbackService,
+                                                                      masterAddress,
+                                                                      workerMessageSender,
                                                                       alertClientService,
                                                                       taskPluginManager,
                                                                       storageOperate));
             if (!offer) {
-                logger.error("submit task to manager error, queue is full, queue size is {}, taskInstanceId: {}",
-                             workerManager.getDelayQueueSize(),
-                             taskExecutionContext.getTaskInstanceId());
+                logger.warn("submit task to wait queue error, queue is full, queue size is {}, taskInstanceId: {}",
+                            workerManager.getDelayQueueSize(),
+                            taskExecutionContext.getTaskInstanceId());
                 taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
-                taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                workerMessageSender.sendMessageWithRetry(taskExecutionContext, masterAddress, CommandType.TASK_EXECUTE_RESULT);
             }
         } finally {
             LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java
deleted file mode 100644
index b62770984a..0000000000
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResponseAckProcessor.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.worker.processor;
-
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.Command;
-import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand;
-import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
-import org.apache.dolphinscheduler.server.worker.cache.ResponseCache;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import com.google.common.base.Preconditions;
-
-import io.netty.channel.Channel;
-
-/**
- * task execute running ack, from master to worker
- */
-@Component
-public class TaskExecuteResponseAckProcessor implements NettyRequestProcessor {
-
-    private final Logger logger = LoggerFactory.getLogger(TaskExecuteResponseAckProcessor.class);
-
-    @Override
-    public void process(Channel channel, Command command) {
-        Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESPONSE_ACK == command.getType(),
-                String.format("invalid command type : %s", command.getType()));
-
-        TaskExecuteResponseAckCommand taskExecuteResponseAckCommand = JSONUtils.parseObject(
-                command.getBody(), TaskExecuteResponseAckCommand.class);
-
-        if (taskExecuteResponseAckCommand == null) {
-            logger.error("task execute response ack command is null");
-            return;
-        }
-        logger.info("task execute response ack command : {}", taskExecuteResponseAckCommand);
-
-        if (taskExecuteResponseAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()) {
-            ResponseCache.get().removeResponseCache(taskExecuteResponseAckCommand.getTaskInstanceId());
-            TaskCallbackService.remove(taskExecuteResponseAckCommand.getTaskInstanceId());
-            logger.debug("remove REMOTE_CHANNELS, task instance id:{}",
-                taskExecuteResponseAckCommand.getTaskInstanceId());
-        } else if (taskExecuteResponseAckCommand.getStatus() == ExecutionStatus.FAILURE.getCode()) {
-            // master handle worker response error, will still retry
-        } else {
-            throw new IllegalArgumentException("Invalid task execute response ack status: "
-                + taskExecuteResponseAckCommand.getStatus());
-        }
-    }
-
-}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java
new file mode 100644
index 0000000000..c60c283c7f
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteResultAckProcessor.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.processor;
+
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand;
+import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
+import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.google.common.base.Preconditions;
+
+import io.netty.channel.Channel;
+
+/**
+ * task execute running ack, from master to worker
+ */
+@Component
+public class TaskExecuteResultAckProcessor implements NettyRequestProcessor {
+
+    private final Logger logger = LoggerFactory.getLogger(TaskExecuteResultAckProcessor.class);
+
+    @Autowired
+    private MessageRetryRunner messageRetryRunner;
+
+    @Override
+    public void process(Channel channel, Command command) {
+        Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESULT_ACK == command.getType(),
+                                    String.format("invalid command type : %s", command.getType()));
+
+        TaskExecuteAckCommand taskExecuteAckMessage = JSONUtils.parseObject(command.getBody(),
+                                                                            TaskExecuteAckCommand.class);
+
+        if (taskExecuteAckMessage == null) {
+            logger.error("task execute response ack command is null");
+            return;
+        }
+        logger.info("task execute response ack command : {}", taskExecuteAckMessage);
+
+        try {
+            LoggerUtils.setTaskInstanceIdMDC(taskExecuteAckMessage.getTaskInstanceId());
+            if (taskExecuteAckMessage.getStatus() == ExecutionStatus.SUCCESS.getCode()) {
+                messageRetryRunner.removeRetryMessage(taskExecuteAckMessage.getTaskInstanceId(),
+                                                      CommandType.TASK_EXECUTE_RESULT);
+                logger.debug("remove REMOTE_CHANNELS, task instance id:{}", taskExecuteAckMessage.getTaskInstanceId());
+            } else if (taskExecuteAckMessage.getStatus() == ExecutionStatus.FAILURE.getCode()) {
+                // master handle worker response error, will still retry
+                logger.error("Receive task execute result ack message, the message status is not success, message: {}",
+                             taskExecuteAckMessage);
+            } else {
+                throw new IllegalArgumentException("Invalid task execute response ack status: "
+                                                       + taskExecuteAckMessage.getStatus());
+            }
+        } finally {
+            LoggerUtils.removeTaskInstanceIdMDC();
+
+        }
+    }
+
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java
index 421e92a530..b85db0cb64 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java
@@ -22,12 +22,13 @@ import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckMessage;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
-import org.apache.dolphinscheduler.server.worker.cache.ResponseCache;
+import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import com.google.common.base.Preconditions;
@@ -42,13 +43,16 @@ public class TaskExecuteRunningAckProcessor implements NettyRequestProcessor {
 
     private final Logger logger = LoggerFactory.getLogger(TaskExecuteRunningAckProcessor.class);
 
+    @Autowired
+    private MessageRetryRunner messageRetryRunner;
+
     @Override
     public void process(Channel channel, Command command) {
         Preconditions.checkArgument(CommandType.TASK_EXECUTE_RUNNING_ACK == command.getType(),
-            String.format("invalid command type : %s", command.getType()));
+                                    String.format("invalid command type : %s", command.getType()));
 
-        TaskExecuteRunningAckCommand runningAckCommand = JSONUtils.parseObject(
-            command.getBody(), TaskExecuteRunningAckCommand.class);
+        TaskExecuteRunningAckMessage runningAckCommand = JSONUtils.parseObject(command.getBody(),
+                                                                               TaskExecuteRunningAckMessage.class);
         if (runningAckCommand == null) {
             logger.error("task execute running ack command is null");
             return;
@@ -58,7 +62,8 @@ public class TaskExecuteRunningAckProcessor implements NettyRequestProcessor {
             logger.info("task execute running ack command : {}", runningAckCommand);
 
             if (runningAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()) {
-                ResponseCache.get().removeRunningCache(runningAckCommand.getTaskInstanceId());
+                messageRetryRunner.removeRetryMessage(runningAckCommand.getTaskInstanceId(),
+                                                      CommandType.TASK_EXECUTE_RUNNING);
             }
         } finally {
             LoggerUtils.removeTaskInstanceIdMDC();
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
index ecc1ffe755..e5633f9aa2 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskKillProcessor.java
@@ -28,17 +28,19 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand;
-import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel;
+import org.apache.dolphinscheduler.remote.command.TaskKillResponseCommand;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.remote.utils.Pair;
 import org.apache.dolphinscheduler.server.utils.ProcessUtils;
+import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner;
 import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
 import org.apache.dolphinscheduler.service.log.LogClientService;
 
 import org.apache.commons.lang.StringUtils;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
@@ -50,6 +52,8 @@ import org.springframework.stereotype.Component;
 import com.google.common.base.Preconditions;
 
 import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
 
 /**
  * task kill processor
@@ -59,18 +63,15 @@ public class TaskKillProcessor implements NettyRequestProcessor {
 
     private final Logger logger = LoggerFactory.getLogger(TaskKillProcessor.class);
 
-    /**
-     * task callback service
-     */
-    @Autowired
-    private TaskCallbackService taskCallbackService;
-
     /**
      * task execute manager
      */
     @Autowired
     private WorkerManagerThread workerManager;
 
+    @Autowired
+    private MessageRetryRunner messageRetryRunner;
+
     /**
      * task kill process
      *
@@ -79,7 +80,8 @@ public class TaskKillProcessor implements NettyRequestProcessor {
      */
     @Override
     public void process(Channel channel, Command command) {
-        Preconditions.checkArgument(CommandType.TASK_KILL_REQUEST == command.getType(), String.format("invalid command type : %s", command.getType()));
+        Preconditions.checkArgument(CommandType.TASK_KILL_REQUEST == command.getType(),
+                                    String.format("invalid command type : %s", command.getType()));
         TaskKillRequestCommand killCommand = JSONUtils.parseObject(command.getBody(), TaskKillRequestCommand.class);
         if (killCommand == null) {
             logger.error("task kill request command is null");
@@ -88,7 +90,8 @@ public class TaskKillProcessor implements NettyRequestProcessor {
         logger.info("task kill command : {}", killCommand);
 
         int taskInstanceId = killCommand.getTaskInstanceId();
-        TaskExecutionContext taskExecutionContext = TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId);
+        TaskExecutionContext taskExecutionContext
+            = TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId);
         if (taskExecutionContext == null) {
             logger.error("taskRequest cache is null, taskInstanceId: {}", killCommand.getTaskInstanceId());
             return;
@@ -98,25 +101,43 @@ public class TaskKillProcessor implements NettyRequestProcessor {
         if (processId == 0) {
             this.cancelApplication(taskInstanceId);
             workerManager.killTaskBeforeExecuteByInstanceId(taskInstanceId);
+            taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.KILL);
             TaskExecutionContextCacheManager.removeByTaskInstanceId(taskInstanceId);
+            sendTaskKillResponseCommand(channel, taskExecutionContext);
             logger.info("the task has not been executed and has been cancelled, task id:{}", taskInstanceId);
             return;
         }
 
         Pair<Boolean, List<String>> result = doKill(taskExecutionContext);
 
-        taskCallbackService.addRemoteChannel(killCommand.getTaskInstanceId(),
-                new NettyRemoteChannel(channel, command.getOpaque()));
-
-        taskExecutionContext.setCurrentExecutionStatus(result.getLeft() ? ExecutionStatus.SUCCESS : ExecutionStatus.FAILURE);
+        taskExecutionContext.setCurrentExecutionStatus(
+            result.getLeft() ? ExecutionStatus.SUCCESS : ExecutionStatus.FAILURE);
         taskExecutionContext.setAppIds(String.join(TaskConstants.COMMA, result.getRight()));
+        sendTaskKillResponseCommand(channel, taskExecutionContext);
 
-        taskCallbackService.sendTaskKillResponseCommand(taskExecutionContext);
         TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+        messageRetryRunner.removeRetryMessages(taskExecutionContext.getTaskInstanceId());
 
         logger.info("remove REMOTE_CHANNELS, task instance id:{}", killCommand.getTaskInstanceId());
     }
 
+    private void sendTaskKillResponseCommand(Channel channel, TaskExecutionContext taskExecutionContext) {
+        TaskKillResponseCommand taskKillResponseCommand = new TaskKillResponseCommand();
+        taskKillResponseCommand.setStatus(taskExecutionContext.getCurrentExecutionStatus().getCode());
+        taskKillResponseCommand.setAppIds(Arrays.asList(taskExecutionContext.getAppIds().split(TaskConstants.COMMA)));
+        taskKillResponseCommand.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+        taskKillResponseCommand.setHost(taskExecutionContext.getHost());
+        taskKillResponseCommand.setProcessId(taskExecutionContext.getProcessId());
+        channel.writeAndFlush(taskKillResponseCommand.convert2Command()).addListener(new ChannelFutureListener() {
+            @Override
+            public void operationComplete(ChannelFuture future) throws Exception {
+                if (!future.isSuccess()) {
+                    logger.error("Submit kill response to master error, kill command: {}", taskKillResponseCommand);
+                }
+            }
+        });
+    }
+
     /**
      * do kill
      *
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRecallCommand.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java
similarity index 100%
rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRecallCommand.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskRejectAckProcessor.java
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java
new file mode 100644
index 0000000000..5a2752ca34
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerMessageSender.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.rpc;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.remote.command.BaseCommand;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
+import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner;
+import org.apache.dolphinscheduler.server.worker.message.MessageSender;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.annotation.PostConstruct;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.stereotype.Component;
+
+import lombok.NonNull;
+
+@Component
+public class WorkerMessageSender {
+
+    private final Logger logger = LoggerFactory.getLogger(WorkerMessageSender.class);
+
+    @Autowired
+    private MessageRetryRunner messageRetryRunner;
+
+    @Autowired
+    private ApplicationContext applicationContext;
+
+    private Map<CommandType, MessageSender> messageSenderMap = new HashMap<>();
+
+    @PostConstruct
+    public void init() {
+        Map<String, MessageSender> messageSenders = applicationContext.getBeansOfType(MessageSender.class);
+        messageSenders.values().forEach(messageSender -> messageSenderMap.put(messageSender.getMessageType(),
+                                                                              messageSender));
+    }
+
+    // todo: use message rather than context
+    public void sendMessageWithRetry(@NonNull TaskExecutionContext taskExecutionContext,
+                                     @NonNull String messageReceiverAddress,
+                                     @NonNull CommandType messageType) {
+        MessageSender messageSender = messageSenderMap.get(messageType);
+        if (messageSender == null) {
+            throw new IllegalArgumentException("The messageType is invalidated, messageType: " + messageType);
+        }
+        BaseCommand baseCommand = messageSender.buildMessage(taskExecutionContext, messageReceiverAddress);
+        try {
+            messageRetryRunner.addRetryMessage(taskExecutionContext.getTaskInstanceId(), messageType, baseCommand);
+            messageSender.sendMessage(baseCommand);
+        } catch (RemotingException e) {
+            logger.error("Send message error, messageType: {}, message: {}", messageType, baseCommand);
+        }
+    }
+
+    public void sendMessage(@NonNull TaskExecutionContext taskExecutionContext,
+                            @NonNull String messageReceiverAddress,
+                            @NonNull CommandType messageType) {
+        MessageSender messageSender = messageSenderMap.get(messageType);
+        if (messageSender == null) {
+            throw new IllegalArgumentException("The messageType is invalidated, messageType: " + messageType);
+        }
+        BaseCommand baseCommand = messageSender.buildMessage(taskExecutionContext, messageReceiverAddress);
+        try {
+            messageSender.sendMessage(baseCommand);
+        } catch (RemotingException e) {
+            logger.error("Send message error, messageType: {}, message: {}", messageType, baseCommand);
+        }
+    }
+
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java
new file mode 100644
index 0000000000..b3cfb1308a
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcClient.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.rpc;
+
+import org.apache.dolphinscheduler.remote.NettyRemotingClient;
+import org.apache.dolphinscheduler.remote.command.Command;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
+import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
+import org.apache.dolphinscheduler.remote.utils.Host;
+import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteResultAckProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteRunningAckProcessor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * This rpc client is only used to send message, will not receive message, all response message should send to {@link WorkerRpcServer}.
+ */
+@Component
+public class WorkerRpcClient implements AutoCloseable {
+
+    private final Logger logger = LoggerFactory.getLogger(WorkerRpcClient.class);
+
+    @Autowired
+    private TaskExecuteRunningAckProcessor taskExecuteRunningAckProcessor;
+
+    @Autowired
+    private TaskExecuteResultAckProcessor taskExecuteResultAckProcessor;
+
+    private NettyRemotingClient nettyRemotingClient;
+
+    public void start() {
+        logger.info("Worker rpc client starting");
+        NettyClientConfig nettyClientConfig = new NettyClientConfig();
+        this.nettyRemotingClient = new NettyRemotingClient(nettyClientConfig);
+        // we only use the client to handle the ack message, we can optimize this, send ack to the nettyServer.
+        this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RUNNING_ACK,
+                                                   taskExecuteRunningAckProcessor);
+        this.nettyRemotingClient.registerProcessor(CommandType.TASK_EXECUTE_RESULT_ACK, taskExecuteResultAckProcessor);
+        logger.info("Worker rpc client started");
+    }
+
+    public void send(Host host, Command command) throws RemotingException {
+        nettyRemotingClient.send(host, command);
+    }
+
+    public void close() {
+        logger.info("Worker rpc client closing");
+        nettyRemotingClient.close();
+        logger.info("Worker rpc client closed");
+    }
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/prc/WorkerRpcServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java
similarity index 88%
rename from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/prc/WorkerRpcServer.java
rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java
index acb0ac6508..e599046c57 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/prc/WorkerRpcServer.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/rpc/WorkerRpcServer.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.server.worker.prc;
+package org.apache.dolphinscheduler.server.worker.rpc;
 
 import org.apache.dolphinscheduler.remote.NettyRemotingServer;
 import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -23,11 +23,11 @@ import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
 import org.apache.dolphinscheduler.server.log.LoggerRequestProcessor;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
 import org.apache.dolphinscheduler.server.worker.processor.HostUpdateProcessor;
-import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor;
-import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteResponseAckProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskDispatchProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteResultAckProcessor;
 import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteRunningAckProcessor;
 import org.apache.dolphinscheduler.server.worker.processor.TaskKillProcessor;
-import org.apache.dolphinscheduler.server.worker.processor.TaskRecallAckProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskRejectAckProcessor;
 
 import java.io.Closeable;
 
@@ -42,19 +42,19 @@ public class WorkerRpcServer implements Closeable {
     private static final Logger LOGGER = LoggerFactory.getLogger(WorkerRpcServer.class);
 
     @Autowired
-    private TaskExecuteProcessor taskExecuteProcessor;
+    private TaskDispatchProcessor taskDispatchProcessor;
 
     @Autowired
     private TaskKillProcessor taskKillProcessor;
 
     @Autowired
-    private TaskRecallAckProcessor taskRecallAckProcessor;
+    private TaskRejectAckProcessor taskRejectAckProcessor;
 
     @Autowired
     private TaskExecuteRunningAckProcessor taskExecuteRunningAckProcessor;
 
     @Autowired
-    private TaskExecuteResponseAckProcessor taskExecuteResponseAckProcessor;
+    private TaskExecuteResultAckProcessor taskExecuteResultAckProcessor;
 
     @Autowired
     private HostUpdateProcessor hostUpdateProcessor;
@@ -72,12 +72,12 @@ public class WorkerRpcServer implements Closeable {
         NettyServerConfig serverConfig = new NettyServerConfig();
         serverConfig.setListenPort(workerConfig.getListenPort());
         this.nettyRemotingServer = new NettyRemotingServer(serverConfig);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_REQUEST, taskExecuteProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_DISPATCH_REQUEST, taskDispatchProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_REQUEST, taskKillProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RUNNING_ACK, taskExecuteRunningAckProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE_ACK, taskExecuteResponseAckProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESULT_ACK, taskExecuteResultAckProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_REJECT_ACK, taskRejectAckProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.PROCESS_HOST_UPDATE_REQUEST, hostUpdateProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_RECALL_ACK, taskRecallAckProcessor);
         // logger server
         this.nettyRemotingServer.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, loggerRequestProcessor);
         this.nettyRemotingServer.registerProcessor(CommandType.ROLL_VIEW_LOG_REQUEST, loggerRequestProcessor);
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
index caa25ebbf5..e69de29bb2 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.worker.runner;
-
-import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
-import org.apache.dolphinscheduler.common.thread.Stopper;
-import org.apache.dolphinscheduler.common.thread.ThreadUtils;
-import org.apache.dolphinscheduler.remote.command.Command;
-import org.apache.dolphinscheduler.server.worker.cache.ResponseCache;
-import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
-
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-/**
- * Retry Report Task Status Thread
- */
-@Component
-public class RetryReportTaskStatusThread extends BaseDaemonThread {
-
-    private final Logger logger = LoggerFactory.getLogger(RetryReportTaskStatusThread.class);
-
-    /**
-     * every 5 minutes
-     */
-    private static long RETRY_REPORT_TASK_STATUS_INTERVAL = 5 * 60 * 1000L;
-
-    @Autowired
-    private TaskCallbackService taskCallbackService;
-
-    protected RetryReportTaskStatusThread() {
-        super("RetryReportTaskStatusThread");
-    }
-
-    @Override
-    public synchronized void start() {
-        logger.info("Retry report task status thread starting");
-        super.start();
-        logger.info("Retry report task status thread started");
-    }
-
-    /**
-     * retry ack/response
-     */
-    @Override
-    public void run() {
-        final ResponseCache instance = ResponseCache.get();
-
-        while (Stopper.isRunning()) {
-
-            // sleep 5 minutes
-            ThreadUtils.sleep(RETRY_REPORT_TASK_STATUS_INTERVAL);
-
-            try {
-                // todo: Only retry the send failed command
-                retryRunningCommand(instance);
-                retryResponseCommand(instance);
-            } catch (Exception e) {
-                logger.warn("Retry report task status error", e);
-            }
-        }
-    }
-
-    private void retryRunningCommand(ResponseCache instance) {
-        if (!instance.getRunningCache().isEmpty()) {
-            Map<Integer, Command> runningCache = instance.getRunningCache();
-            logger.info("Send task running retry command starting, waiting to retry size: {}", runningCache.size());
-            for (Map.Entry<Integer, Command> entry : runningCache.entrySet()) {
-                Integer taskInstanceId = entry.getKey();
-                Command runningCommand = entry.getValue();
-                try {
-                    taskCallbackService.send(taskInstanceId, runningCommand);
-                } catch (Exception ex) {
-                    logger.error("Retry send running command to master error, taskInstanceId: {}, command: {}", taskInstanceId, runningCommand);
-                }
-            }
-            logger.info("Send task running retry command finished, waiting to retry size: {}", runningCache.size());
-        }
-    }
-
-    private void retryResponseCommand(ResponseCache instance) {
-        Map<Integer, Command> responseCache = instance.getResponseCache();
-        if (!responseCache.isEmpty()) {
-            logger.info("Send task response retry command starting, waiting to retry size: {}", responseCache.size());
-            for (Map.Entry<Integer, Command> entry : responseCache.entrySet()) {
-                Integer taskInstanceId = entry.getKey();
-                Command responseCommand = entry.getValue();
-                try {
-                    taskCallbackService.send(taskInstanceId, responseCommand);
-                } catch (Exception ex) {
-                    logger.error("Retry send response command to master error, taskInstanceId: {}, command: {}", taskInstanceId, responseCommand);
-                }
-            }
-            logger.info("Send task response retry command finished, waiting to retry size: {}", responseCache.size());
-        }
-    }
-
-}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
index b5fecf6c02..86bd34061b 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@@ -35,8 +35,9 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheMana
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.plugin.task.api.model.Property;
 import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo;
+import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.server.utils.ProcessUtils;
-import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
 import org.apache.dolphinscheduler.service.exceptions.ServiceException;
 import org.apache.dolphinscheduler.service.task.TaskPluginManager;
@@ -61,6 +62,8 @@ import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import io.micrometer.core.lang.NonNull;
+
 /**
  * task scheduler thread
  */
@@ -74,9 +77,11 @@ public class TaskExecuteThread implements Runnable, Delayed {
     /**
      * task instance
      */
-    private TaskExecutionContext taskExecutionContext;
+    private final TaskExecutionContext taskExecutionContext;
+
+    private final String masterAddress;
 
-    private StorageOperate storageOperate;
+    private final StorageOperate storageOperate;
 
     /**
      * abstract task
@@ -86,12 +91,12 @@ public class TaskExecuteThread implements Runnable, Delayed {
     /**
      * task callback service
      */
-    private TaskCallbackService taskCallbackService;
+    private final WorkerMessageSender workerMessageSender;
 
     /**
      * alert client server
      */
-    private AlertClientService alertClientService;
+    private final AlertClientService alertClientService;
 
     private TaskPluginManager taskPluginManager;
 
@@ -99,25 +104,29 @@ public class TaskExecuteThread implements Runnable, Delayed {
      * constructor
      *
      * @param taskExecutionContext taskExecutionContext
-     * @param taskCallbackService  taskCallbackService
+     * @param workerMessageSender  used for worker send message to master
      */
-    public TaskExecuteThread(TaskExecutionContext taskExecutionContext,
-                             TaskCallbackService taskCallbackService,
-                             AlertClientService alertClientService,
+    public TaskExecuteThread(@NonNull TaskExecutionContext taskExecutionContext,
+                             @NonNull String masterAddress,
+                             @NonNull WorkerMessageSender workerMessageSender,
+                             @NonNull AlertClientService alertClientService,
                              StorageOperate storageOperate) {
         this.taskExecutionContext = taskExecutionContext;
-        this.taskCallbackService = taskCallbackService;
+        this.masterAddress = masterAddress;
+        this.workerMessageSender = workerMessageSender;
         this.alertClientService = alertClientService;
         this.storageOperate = storageOperate;
     }
 
-    public TaskExecuteThread(TaskExecutionContext taskExecutionContext,
-                             TaskCallbackService taskCallbackService,
-                             AlertClientService alertClientService,
-                             TaskPluginManager taskPluginManager,
+    public TaskExecuteThread(@NonNull TaskExecutionContext taskExecutionContext,
+                             @NonNull String masterAddress,
+                             @NonNull WorkerMessageSender workerMessageSender,
+                             @NonNull AlertClientService alertClientService,
+                             @NonNull TaskPluginManager taskPluginManager,
                              StorageOperate storageOperate) {
         this.taskExecutionContext = taskExecutionContext;
-        this.taskCallbackService = taskCallbackService;
+        this.masterAddress = masterAddress;
+        this.workerMessageSender = workerMessageSender;
         this.alertClientService = alertClientService;
         this.taskPluginManager = taskPluginManager;
         this.storageOperate = storageOperate;
@@ -125,18 +134,26 @@ public class TaskExecuteThread implements Runnable, Delayed {
 
     @Override
     public void run() {
-        if (Constants.DRY_RUN_FLAG_YES == taskExecutionContext.getDryRun()) {
-            taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.SUCCESS);
-            taskExecutionContext.setStartTime(new Date());
-            taskExecutionContext.setEndTime(new Date());
-            TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-            taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
-            logger.info("[WorkflowInstance-{}][TaskInstance-{}] Task dry run success",
-                taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId());
-            return;
+        try {
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(),
+                                                        taskExecutionContext.getTaskInstanceId());
+            if (Constants.DRY_RUN_FLAG_YES == taskExecutionContext.getDryRun()) {
+                taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.SUCCESS);
+                taskExecutionContext.setStartTime(new Date());
+                taskExecutionContext.setEndTime(new Date());
+                TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+                workerMessageSender.sendMessageWithRetry(taskExecutionContext,
+                                                         masterAddress,
+                                                         CommandType.TASK_EXECUTE_RESULT);
+                logger.info("Task dry run success");
+                return;
+            }
+        } finally {
+            LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
         }
         try {
-            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId());
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(),
+                                                        taskExecutionContext.getTaskInstanceId());
             logger.info("script path : {}", taskExecutionContext.getExecutePath());
             if (taskExecutionContext.getStartTime() == null) {
                 taskExecutionContext.setStartTime(new Date());
@@ -145,10 +162,13 @@ public class TaskExecuteThread implements Runnable, Delayed {
 
             // callback task execute running
             taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.RUNNING_EXECUTION);
-            taskCallbackService.sendTaskExecuteRunningCommand(taskExecutionContext);
+            workerMessageSender.sendMessageWithRetry(taskExecutionContext,
+                                                     masterAddress,
+                                                     CommandType.TASK_EXECUTE_RUNNING);
 
             // copy hdfs/minio file to local
-            List<Pair<String, String>> fileDownloads = downloadCheck(taskExecutionContext.getExecutePath(), taskExecutionContext.getResources());
+            List<Pair<String, String>> fileDownloads = downloadCheck(taskExecutionContext.getExecutePath(),
+                                                                     taskExecutionContext.getResources());
             if (!fileDownloads.isEmpty()) {
                 downloadResource(taskExecutionContext.getExecutePath(), logger, fileDownloads);
             }
@@ -157,8 +177,8 @@ public class TaskExecuteThread implements Runnable, Delayed {
             taskExecutionContext.setDefinedParams(getGlobalParamsMap());
 
             taskExecutionContext.setTaskAppId(String.format("%s_%s",
-                    taskExecutionContext.getProcessInstanceId(),
-                    taskExecutionContext.getTaskInstanceId()));
+                                                            taskExecutionContext.getProcessInstanceId(),
+                                                            taskExecutionContext.getTaskInstanceId()));
 
             preBuildBusinessParams();
 
@@ -207,7 +227,9 @@ public class TaskExecuteThread implements Runnable, Delayed {
             taskExecutionContext.setAppIds(this.task.getAppIds());
         } finally {
             TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-            taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+            workerMessageSender.sendMessageWithRetry(taskExecutionContext,
+                                                     masterAddress,
+                                                     CommandType.TASK_EXECUTE_RESULT);
             clearTaskExecPath();
             LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
         }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java
index 67abf523ed..98de315345 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java
@@ -68,8 +68,10 @@ public class WorkerExecService {
 
             @Override
             public void onFailure(Throwable throwable) {
-                logger.error("task execute failed, processInstanceId:{}, taskInstanceId:{}", taskExecuteThread.getTaskExecutionContext().getProcessInstanceId()
-                    , taskExecuteThread.getTaskExecutionContext().getTaskInstanceId(), throwable);
+                logger.error("task execute failed, processInstanceId:{}, taskInstanceId:{}",
+                             taskExecuteThread.getTaskExecutionContext().getProcessInstanceId(),
+                             taskExecuteThread.getTaskExecutionContext().getTaskInstanceId(),
+                             throwable);
                 taskExecuteThreadMap.remove(taskExecuteThread.getTaskExecutionContext().getTaskInstanceId());
             }
         };
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
index 9c826e0d5a..92b7c272d2 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
@@ -20,11 +20,8 @@ package org.apache.dolphinscheduler.server.worker.runner;
 import org.apache.dolphinscheduler.common.storage.StorageOperate;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
-import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender;
 
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.DelayQueue;
@@ -45,7 +42,7 @@ public class WorkerManagerThread implements Runnable {
     /**
      * task queue
      */
-    private final DelayQueue<TaskExecuteThread> workerExecuteQueue = new DelayQueue<>();
+    private final DelayQueue<TaskExecuteThread> waitSubmitQueue = new DelayQueue<>();
 
     @Autowired(required = false)
     private StorageOperate storageOperate;
@@ -59,7 +56,7 @@ public class WorkerManagerThread implements Runnable {
      * task callback service
      */
     @Autowired
-    private TaskCallbackService taskCallbackService;
+    private WorkerMessageSender workerMessageSender;
 
     /**
      * running task
@@ -83,7 +80,7 @@ public class WorkerManagerThread implements Runnable {
      * @return queue size
      */
     public int getDelayQueueSize() {
-        return workerExecuteQueue.size();
+        return waitSubmitQueue.size();
     }
 
     /**
@@ -100,23 +97,12 @@ public class WorkerManagerThread implements Runnable {
      * then send Response to Master, update the execution status of task instance
      */
     public void killTaskBeforeExecuteByInstanceId(Integer taskInstanceId) {
-        workerExecuteQueue.stream()
-                          .filter(taskExecuteThread -> taskExecuteThread.getTaskExecutionContext().getTaskInstanceId() == taskInstanceId)
-                          .forEach(workerExecuteQueue::remove);
-        sendTaskKillResponse(taskInstanceId);
+        waitSubmitQueue.stream()
+            .filter(taskExecuteThread -> taskExecuteThread.getTaskExecutionContext().getTaskInstanceId()
+                == taskInstanceId)
+                          .forEach(waitSubmitQueue::remove);
     }
 
-    /**
-     * kill task before execute , like delay task
-     */
-    private void sendTaskKillResponse(Integer taskInstanceId) {
-        TaskExecutionContext taskExecutionContext = TaskExecutionContextCacheManager.getByTaskInstanceId(taskInstanceId);
-        if (taskExecutionContext == null) {
-            return;
-        }
-        taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.KILL);
-        taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
-    }
 
     /**
      * submit task
@@ -125,7 +111,7 @@ public class WorkerManagerThread implements Runnable {
      * @return submit result
      */
     public boolean offer(TaskExecuteThread taskExecuteThread) {
-        return workerExecuteQueue.offer(taskExecuteThread);
+        return waitSubmitQueue.offer(taskExecuteThread);
     }
 
     public void start() {
@@ -142,7 +128,7 @@ public class WorkerManagerThread implements Runnable {
         TaskExecuteThread taskExecuteThread;
         while (Stopper.isRunning()) {
             try {
-                taskExecuteThread = workerExecuteQueue.take();
+                taskExecuteThread = waitSubmitQueue.take();
                 workerExecService.submit(taskExecuteThread);
             } catch (Exception e) {
                 logger.error("An unexpected interrupt is happened, "
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java
similarity index 61%
rename from dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
rename to dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java
index 2187a19281..08db9060a6 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java
@@ -25,11 +25,12 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRequestCommand;
+import org.apache.dolphinscheduler.remote.command.TaskDispatchCommand;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningCommand;
 import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
 import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender;
 import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
@@ -48,19 +49,20 @@ import org.powermock.api.mockito.PowerMockito;
 import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * test task execute processor
  */
 @RunWith(PowerMockRunner.class)
-@PrepareForTest({SpringApplicationContext.class, TaskCallbackService.class, WorkerConfig.class, FileUtils.class,
-    JsonSerializer.class, JSONUtils.class, ThreadUtils.class, ExecutorService.class, ChannelUtils.class})
+@PrepareForTest({SpringApplicationContext.class, WorkerConfig.class, FileUtils.class, JsonSerializer.class,
+    JSONUtils.class, ThreadUtils.class, ExecutorService.class, ChannelUtils.class})
 @Ignore
-public class TaskExecuteProcessorTest {
+public class TaskDispatchProcessorTest {
 
     private TaskExecutionContext taskExecutionContext;
 
-    private TaskCallbackService taskCallbackService;
+    private WorkerMessageSender workerMessageSender;
 
     private ExecutorService workerExecService;
 
@@ -72,7 +74,7 @@ public class TaskExecuteProcessorTest {
 
     private Command ackCommand;
 
-    private TaskExecuteRequestCommand taskRequestCommand;
+    private TaskDispatchCommand taskRequestCommand;
 
     private AlertClientService alertClientService;
 
@@ -86,66 +88,73 @@ public class TaskExecuteProcessorTest {
         workerConfig.setExecThreads(1);
         workerConfig.setListenPort(1234);
         command = new Command();
-        command.setType(CommandType.TASK_EXECUTE_REQUEST);
-        ackCommand = new TaskExecuteRunningCommand().convert2Command();
-        taskRequestCommand = new TaskExecuteRequestCommand(taskExecutionContext);
+        command.setType(CommandType.TASK_DISPATCH_REQUEST);
+        ackCommand = new TaskExecuteRunningCommand("127.0.0.1:1234",
+                                                   "127.0.0.1:5678",
+                                                   System.currentTimeMillis()).convert2Command();
+        taskRequestCommand = new TaskDispatchCommand(taskExecutionContext,
+                                                     "127.0.0.1:5678",
+                                                     "127.0.0.1:1234",
+                                                     System.currentTimeMillis());
         alertClientService = PowerMockito.mock(AlertClientService.class);
         workerExecService = PowerMockito.mock(ExecutorService.class);
-        PowerMockito.when(workerExecService.submit(Mockito.any(TaskExecuteThread.class)))
-                .thenReturn(null);
+        PowerMockito.when(workerExecService.submit(Mockito.any(TaskExecuteThread.class))).thenReturn(null);
 
         PowerMockito.mockStatic(ChannelUtils.class);
         PowerMockito.when(ChannelUtils.toAddress(null)).thenReturn(null);
 
-        taskCallbackService = PowerMockito.mock(TaskCallbackService.class);
-        PowerMockito.doNothing().when(taskCallbackService).send(taskExecutionContext.getTaskInstanceId(), ackCommand);
+        workerMessageSender = PowerMockito.mock(WorkerMessageSender.class);
 
         PowerMockito.mockStatic(SpringApplicationContext.class);
-        PowerMockito.when(SpringApplicationContext.getBean(TaskCallbackService.class))
-            .thenReturn(taskCallbackService);
-        PowerMockito.when(SpringApplicationContext.getBean(WorkerConfig.class))
-            .thenReturn(workerConfig);
+        PowerMockito.when(SpringApplicationContext.getBean(WorkerMessageSender.class)).thenReturn(workerMessageSender);
+        PowerMockito.when(SpringApplicationContext.getBean(WorkerConfig.class)).thenReturn(workerConfig);
 
         workerManager = PowerMockito.mock(WorkerManagerThread.class);
 
         storageOperate = PowerMockito.mock(StorageOperate.class);
-        PowerMockito.when(
-                workerManager.offer(new TaskExecuteThread(taskExecutionContext, taskCallbackService, alertClientService, storageOperate)))
-            .thenReturn(Boolean.TRUE);
+        PowerMockito.when(workerManager.offer(new TaskExecuteThread(taskExecutionContext,
+                                                                    "127.0.0.1:5678",
+                                                                    workerMessageSender,
+                                                                    alertClientService,
+                                                                    storageOperate))).thenReturn(Boolean.TRUE);
 
-        PowerMockito.when(SpringApplicationContext.getBean(WorkerManagerThread.class))
-            .thenReturn(workerManager);
+        PowerMockito.when(SpringApplicationContext.getBean(WorkerManagerThread.class)).thenReturn(workerManager);
 
         PowerMockito.mockStatic(ThreadUtils.class);
-        PowerMockito.when(ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", workerConfig.getExecThreads()))
-            .thenReturn(workerExecService);
+        PowerMockito.when(ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread",
+                                                                   workerConfig.getExecThreads())).thenReturn(
+            workerExecService);
 
         PowerMockito.mockStatic(JsonSerializer.class);
-        PowerMockito.when(JsonSerializer.deserialize(command.getBody(), TaskExecuteRequestCommand.class))
-                .thenReturn(taskRequestCommand);
+        PowerMockito.when(JsonSerializer.deserialize(command.getBody(), TaskDispatchCommand.class)).thenReturn(
+            taskRequestCommand);
 
         PowerMockito.mockStatic(JSONUtils.class);
-        PowerMockito.when(JSONUtils.parseObject(command.getBody(), TaskExecuteRequestCommand.class))
-                .thenReturn(taskRequestCommand);
+        PowerMockito.when(JSONUtils.parseObject(command.getBody(), TaskDispatchCommand.class)).thenReturn(
+            taskRequestCommand);
 
         PowerMockito.mockStatic(FileUtils.class);
         PowerMockito.when(FileUtils.getProcessExecDir(taskExecutionContext.getProjectCode(),
-                taskExecutionContext.getProcessDefineCode(),
-                taskExecutionContext.getProcessDefineVersion(),
-                taskExecutionContext.getProcessInstanceId(),
-                taskExecutionContext.getTaskInstanceId()))
-            .thenReturn(taskExecutionContext.getExecutePath());
+                                                      taskExecutionContext.getProcessDefineCode(),
+                                                      taskExecutionContext.getProcessDefineVersion(),
+                                                      taskExecutionContext.getProcessInstanceId(),
+                                                      taskExecutionContext.getTaskInstanceId())).thenReturn(
+            taskExecutionContext.getExecutePath());
         PowerMockito.doNothing().when(FileUtils.class, "createWorkDirIfAbsent", taskExecutionContext.getExecutePath());
 
-        SimpleTaskExecuteThread simpleTaskExecuteThread = new SimpleTaskExecuteThread(
-            null, null, null, alertClientService, storageOperate);
-        PowerMockito.whenNew(TaskExecuteThread.class).withAnyArguments()
-            .thenReturn(simpleTaskExecuteThread);
+        SimpleTaskExecuteThread simpleTaskExecuteThread = new SimpleTaskExecuteThread(new TaskExecutionContext(),
+                                                                                      workerMessageSender,
+                                                                                      "127.0.0.1:5678",
+                                                                                      LoggerFactory.getLogger(
+                                                                                          TaskDispatchProcessorTest.class),
+                                                                                      alertClientService,
+                                                                                      storageOperate);
+        PowerMockito.whenNew(TaskExecuteThread.class).withAnyArguments().thenReturn(simpleTaskExecuteThread);
     }
 
     @Test
     public void testNormalExecution() {
-        TaskExecuteProcessor processor = new TaskExecuteProcessor();
+        TaskDispatchProcessor processor = new TaskDispatchProcessor();
         processor.process(null, command);
 
         Assert.assertEquals(ExecutionStatus.RUNNING_EXECUTION, taskExecutionContext.getCurrentExecutionStatus());
@@ -154,7 +163,7 @@ public class TaskExecuteProcessorTest {
     @Test
     public void testDelayExecution() {
         taskExecutionContext.setDelayTime(1);
-        TaskExecuteProcessor processor = new TaskExecuteProcessor();
+        TaskDispatchProcessor processor = new TaskDispatchProcessor();
         processor.process(null, command);
 
         Assert.assertEquals(ExecutionStatus.DELAY_EXECUTION, taskExecutionContext.getCurrentExecutionStatus());
@@ -179,11 +188,12 @@ public class TaskExecuteProcessorTest {
     private static class SimpleTaskExecuteThread extends TaskExecuteThread {
 
         public SimpleTaskExecuteThread(TaskExecutionContext taskExecutionContext,
-                                       TaskCallbackService taskCallbackService,
+                                       WorkerMessageSender workerMessageSender,
+                                       String masterAddress,
                                        Logger taskLogger,
                                        AlertClientService alertClientService,
                                        StorageOperate storageOperate) {
-            super(taskExecutionContext, taskCallbackService, alertClientService, storageOperate);
+            super(taskExecutionContext, masterAddress, workerMessageSender, alertClientService, storageOperate);
         }
 
         @Override
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java
index 90577111b9..bd492c7ff6 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java
@@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.server.worker.runner;
 
 import org.apache.dolphinscheduler.common.storage.StorageOperate;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
 import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClientTest;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
 import org.apache.dolphinscheduler.service.task.TaskPluginManager;
 
@@ -48,7 +48,7 @@ public class TaskExecuteThreadTest {
     private TaskExecutionContext taskExecutionContext;
 
     @Mock
-    private TaskCallbackService taskCallbackService;
+    private WorkerMessageSender workerMessageSender;
 
     @Mock
     private AlertClientService alertClientService;
@@ -61,8 +61,12 @@ public class TaskExecuteThreadTest {
 
     @Test
     public void checkTest() {
-        TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService,
-            alertClientService, taskPluginManager, storageOperate);
+        TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext,
+                                                                    "127.0.0.1:5678",
+                                                                    workerMessageSender,
+                                                                    alertClientService,
+                                                                    taskPluginManager,
+                                                                    storageOperate);
 
         String path = "/";
         Map<String, String> projectRes = new HashMap<>();


[dolphinscheduler] 15/29: [Improvement-10617] Add comment in slot check (#10618)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 6c83967ebecc0fe0bb07cdeb09c4f42e2321dbe1
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Thu Jun 30 15:58:49 2022 +0800

    [Improvement-10617] Add comment in slot check (#10618)
    
    (cherry picked from commit 247ca4ae8a6d6e7a0580a999e04692a69f885144)
---
 .../server/master/runner/MasterSchedulerService.java                 | 5 +++--
 .../apache/dolphinscheduler/service/process/ProcessServiceImpl.java  | 4 +++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index 6de3541dae..1d0071a4dd 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -202,8 +202,9 @@ public class MasterSchedulerService extends BaseDaemonThread {
         for (final Command command : commands) {
             masterPrepareExecService.execute(() -> {
                 try {
-                    // todo: this check is not safe, the slot may change after command transform.
-                    // slot check again
+                    // Note: this check is not safe, the slot may change after command transform.
+                    // We use the database transaction in `handleCommand` so that we can guarantee the command will always be executed
+                    // by only one master
                     SlotCheckState slotCheckState = slotCheck(command);
                     if (slotCheckState.equals(SlotCheckState.CHANGE) || slotCheckState.equals(SlotCheckState.INJECT)) {
                         logger.info("Master handle command {} skip, slot check state: {}", command.getId(), slotCheckState);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 965ab89801..c527276933 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -137,6 +137,7 @@ import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.math.NumberUtils;
 
+import java.sql.SQLIntegrityConstraintViolationException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
@@ -350,7 +351,8 @@ public class ProcessServiceImpl implements ProcessService {
     }
 
     /**
-     * save error command, and delete original command
+     * Save error command, and delete original command. If the given command has already been moved into error command,
+     * will throw {@link SQLIntegrityConstraintViolationException ).
      *
      * @param command command
      * @param message message


[dolphinscheduler] 14/29: Validate master/worker config (#10649)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 4b224ae2e513b1124a0abaa4b5cde4b964d27a28
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jun 28 20:17:43 2022 +0800

    Validate master/worker config (#10649)
    
    (cherry picked from commit 35b25da863e38df817bfe3cb2ef758d426b8b699)
---
 .../server/master/config/MasterConfig.java         | 206 +++++++--------------
 .../master/registry/MasterRegistryClient.java      |   5 +-
 .../master/runner/FailoverExecuteThread.java       |   2 +-
 .../master/runner/StateWheelExecuteThread.java     |   2 +-
 .../master/runner/task/BaseTaskProcessor.java      |   5 +-
 .../src/main/resources/application.yaml            |  10 +-
 .../server/master/BlockingTaskTest.java            |   5 +-
 .../server/master/ConditionsTaskTest.java          |   3 +-
 .../server/master/DependentTaskTest.java           |   3 +-
 .../server/master/SubProcessTaskTest.java          |   3 +-
 .../server/master/SwitchTaskTest.java              |   3 +-
 .../service/process/ProcessService.java            |   2 +-
 .../service/process/ProcessServiceImpl.java        |   2 +-
 .../src/main/resources/application.yaml            |  16 +-
 .../server/worker/config/WorkerConfig.java         | 131 ++++---------
 .../worker/registry/WorkerRegistryClient.java      |   2 +-
 .../src/main/resources/application.yaml            |   4 +-
 .../worker/registry/WorkerRegistryClientTest.java  |  13 +-
 18 files changed, 156 insertions(+), 261 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
index 76d4ae1525..0494b7dd00 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
@@ -21,170 +21,108 @@ import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelect
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
-import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.stereotype.Component;
+import java.time.Duration;
 
-@Component
-@EnableConfigurationProperties
-@ConfigurationProperties("master")
-public class MasterConfig {
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.validation.Errors;
+import org.springframework.validation.Validator;
+import org.springframework.validation.annotation.Validated;
+
+import lombok.Data;
+
+@Data
+@Validated
+@Configuration
+@ConfigurationProperties(prefix = "master")
+public class MasterConfig implements Validator {
     /**
      * The master RPC server listen port.
      */
-    private int listenPort;
+    private int listenPort = 5678;
     /**
      * The max batch size used to fetch command from database.
      */
-    private int fetchCommandNum;
+    private int fetchCommandNum = 10;
     /**
      * The thread number used to prepare processInstance. This number shouldn't bigger than fetchCommandNum.
      */
-    private int preExecThreads;
+    private int preExecThreads = 10;
     /**
      * todo: We may need to split the process/task into different thread size.
      * The thread number used to handle processInstance and task event.
      * Will create two thread poll to execute {@link WorkflowExecuteRunnable} and {@link TaskExecuteRunnable}.
      */
-    private int execThreads;
+    private int execThreads = 10;
     /**
      * The task dispatch thread pool size.
      */
-    private int dispatchTaskNumber;
+    private int dispatchTaskNumber = 3;
     /**
      * Worker select strategy.
      */
-    private HostSelector hostSelector;
+    private HostSelector hostSelector = HostSelector.LOWER_WEIGHT;
     /**
      * Master heart beat task execute interval.
      */
-    private int heartbeatInterval;
+    private Duration heartbeatInterval = Duration.ofSeconds(10);
     /**
      * task submit max retry times.
      */
-    private int taskCommitRetryTimes;
+    private int taskCommitRetryTimes = 5;
     /**
-     * task submit retry interval/ms.
+     * task submit retry interval.
      */
-    private int taskCommitInterval;
+    private Duration taskCommitInterval = Duration.ofSeconds(1);
     /**
-     * state wheel check interval/ms, if this value is bigger, may increase the delay of task/processInstance.
+     * state wheel check interval, if this value is bigger, may increase the delay of task/processInstance.
      */
-    private int stateWheelInterval;
-    private double maxCpuLoadAvg;
-    private double reservedMemory;
-    private int failoverInterval;
-    private boolean killYarnJobWhenTaskFailover;
-
-    public int getListenPort() {
-        return listenPort;
-    }
-
-    public void setListenPort(int listenPort) {
-        this.listenPort = listenPort;
-    }
-
-    public int getFetchCommandNum() {
-        return fetchCommandNum;
-    }
-
-    public void setFetchCommandNum(int fetchCommandNum) {
-        this.fetchCommandNum = fetchCommandNum;
-    }
-
-    public int getPreExecThreads() {
-        return preExecThreads;
-    }
-
-    public void setPreExecThreads(int preExecThreads) {
-        this.preExecThreads = preExecThreads;
-    }
-
-    public int getExecThreads() {
-        return execThreads;
-    }
-
-    public void setExecThreads(int execThreads) {
-        this.execThreads = execThreads;
-    }
-
-    public int getDispatchTaskNumber() {
-        return dispatchTaskNumber;
-    }
-
-    public void setDispatchTaskNumber(int dispatchTaskNumber) {
-        this.dispatchTaskNumber = dispatchTaskNumber;
-    }
-
-    public HostSelector getHostSelector() {
-        return hostSelector;
-    }
-
-    public void setHostSelector(HostSelector hostSelector) {
-        this.hostSelector = hostSelector;
-    }
-
-    public int getHeartbeatInterval() {
-        return heartbeatInterval;
-    }
-
-    public void setHeartbeatInterval(int heartbeatInterval) {
-        this.heartbeatInterval = heartbeatInterval;
-    }
-
-    public int getTaskCommitRetryTimes() {
-        return taskCommitRetryTimes;
-    }
-
-    public void setTaskCommitRetryTimes(int taskCommitRetryTimes) {
-        this.taskCommitRetryTimes = taskCommitRetryTimes;
-    }
-
-    public int getTaskCommitInterval() {
-        return taskCommitInterval;
-    }
-
-    public void setTaskCommitInterval(int taskCommitInterval) {
-        this.taskCommitInterval = taskCommitInterval;
-    }
-
-    public int getStateWheelInterval() {
-        return stateWheelInterval;
-    }
-
-    public void setStateWheelInterval(int stateWheelInterval) {
-        this.stateWheelInterval = stateWheelInterval;
-    }
-
-    public double getMaxCpuLoadAvg() {
-        return maxCpuLoadAvg > 0 ? maxCpuLoadAvg : Runtime.getRuntime().availableProcessors() * 2;
-    }
-
-    public void setMaxCpuLoadAvg(double maxCpuLoadAvg) {
-        this.maxCpuLoadAvg = maxCpuLoadAvg;
-    }
-
-    public double getReservedMemory() {
-        return reservedMemory;
-    }
-
-    public void setReservedMemory(double reservedMemory) {
-        this.reservedMemory = reservedMemory;
-    }
-
-    public int getFailoverInterval() {
-        return failoverInterval;
-    }
-
-    public void setFailoverInterval(int failoverInterval) {
-        this.failoverInterval = failoverInterval;
-    }
-
-    public boolean isKillYarnJobWhenTaskFailover() {
-        return killYarnJobWhenTaskFailover;
-    }
-
-    public void setKillYarnJobWhenTaskFailover(boolean killYarnJobWhenTaskFailover) {
-        this.killYarnJobWhenTaskFailover = killYarnJobWhenTaskFailover;
+    private Duration stateWheelInterval = Duration.ofMillis(5);
+    private double maxCpuLoadAvg = -1;
+    private double reservedMemory = 0.3;
+    private Duration failoverInterval = Duration.ofMinutes(10);
+    private boolean killYarnJobWhenTaskFailover = true;
+
+    @Override
+    public boolean supports(Class<?> clazz) {
+        return MasterConfig.class.isAssignableFrom(clazz);
+    }
+
+    @Override
+    public void validate(Object target, Errors errors) {
+        MasterConfig masterConfig = (MasterConfig) target;
+        if (masterConfig.getListenPort() <= 0) {
+            errors.rejectValue("listen-port", null, "is invalidated");
+        }
+        if (masterConfig.getFetchCommandNum() <= 0) {
+            errors.rejectValue("fetch-command-num", null, "should be a positive value");
+        }
+        if (masterConfig.getPreExecThreads() <= 0) {
+            errors.rejectValue("per-exec-threads", null, "should be a positive value");
+        }
+        if (masterConfig.getExecThreads() <= 0) {
+            errors.rejectValue("exec-threads", null, "should be a positive value");
+        }
+        if (masterConfig.getDispatchTaskNumber() <= 0) {
+            errors.rejectValue("dispatch-task-number", null, "should be a positive value");
+        }
+        if (masterConfig.getHeartbeatInterval().toMillis() < 0) {
+            errors.rejectValue("heartbeat-interval", null, "should be a valid duration");
+        }
+        if (masterConfig.getTaskCommitRetryTimes() <= 0) {
+            errors.rejectValue("task-commit-retry-times", null, "should be a positive value");
+        }
+        if (masterConfig.getTaskCommitInterval().toMillis() <= 0) {
+            errors.rejectValue("task-commit-interval", null, "should be a valid duration");
+        }
+        if (masterConfig.getStateWheelInterval().toMillis() <= 0) {
+            errors.rejectValue("state-wheel-interval", null, "should be a valid duration");
+        }
+        if (masterConfig.getFailoverInterval().toMillis() <= 0) {
+            errors.rejectValue("failover-interval", null, "should be a valid duration");
+        }
+        if (masterConfig.getMaxCpuLoadAvg() <= 0) {
+            masterConfig.setMaxCpuLoadAvg(Runtime.getRuntime().availableProcessors() * 2);
+        }
     }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
index 771bac4a7a..486c360117 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
@@ -35,6 +35,7 @@ import org.apache.dolphinscheduler.service.registry.RegistryClient;
 
 import org.apache.commons.lang3.StringUtils;
 
+import java.time.Duration;
 import java.util.Collections;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -187,7 +188,7 @@ public class MasterRegistryClient {
     void registry() {
         logger.info("Master node : {} registering to registry center", masterAddress);
         String localNodePath = getCurrentNodePath();
-        int masterHeartbeatInterval = masterConfig.getHeartbeatInterval();
+        Duration masterHeartbeatInterval = masterConfig.getHeartbeatInterval();
         HeartBeatTask heartBeatTask = new HeartBeatTask(startupTime,
                                                         masterConfig.getMaxCpuLoadAvg(),
                                                         masterConfig.getReservedMemory(),
@@ -210,7 +211,7 @@ public class MasterRegistryClient {
         // delete dead server
         registryClient.handleDeadServer(Collections.singleton(localNodePath), NodeType.MASTER, Constants.DELETE_OP);
 
-        this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, 0L, masterHeartbeatInterval, TimeUnit.SECONDS);
+        this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, 0L, masterHeartbeatInterval.getSeconds(), TimeUnit.SECONDS);
         logger.info("Master node : {} registered to registry center successfully with heartBeatInterval : {}s", masterAddress, masterHeartbeatInterval);
 
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
index d6f3937f4b..63f4215f27 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
@@ -67,7 +67,7 @@ public class FailoverExecuteThread extends BaseDaemonThread {
             } catch (Exception e) {
                 logger.error("Master failover thread execute error", e);
             } finally {
-                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * masterConfig.getFailoverInterval() * 60);
+                ThreadUtils.sleep(masterConfig.getFailoverInterval().toMillis());
             }
         }
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 65c7db924d..8b92696723 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -93,7 +93,7 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
 
     @Override
     public void run() {
-        Duration checkInterval = Duration.ofMillis(masterConfig.getStateWheelInterval() * Constants.SLEEP_TIME_MILLIS);
+        Duration checkInterval = masterConfig.getStateWheelInterval();
         while (Stopper.isRunning()) {
             try {
                 checkTask4Timeout();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
index 12c3332961..4a91d8d7dc 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
@@ -63,6 +63,7 @@ import org.apache.dolphinscheduler.spi.utils.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -107,7 +108,7 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
 
     protected int maxRetryTimes;
 
-    protected int commitInterval;
+    protected long commitInterval;
 
     protected ProcessService processService;
 
@@ -125,7 +126,7 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
         this.taskInstance = taskInstance;
         this.processInstance = processInstance;
         this.maxRetryTimes = masterConfig.getTaskCommitRetryTimes();
-        this.commitInterval = masterConfig.getTaskCommitInterval();
+        this.commitInterval = masterConfig.getTaskCommitInterval().toMillis();
     }
 
     protected javax.sql.DataSource defaultDataSource =
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index 65163cdb3c..a277272026 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -96,19 +96,19 @@ master:
   dispatch-task-number: 3
   # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
   host-selector: lower_weight
-  # master heartbeat interval, the unit is second
-  heartbeat-interval: 10
+  # master heartbeat interval
+  heartbeat-interval: 10s
   # master commit task retry times
   task-commit-retry-times: 5
-  # master commit task interval, the unit is millisecond
-  task-commit-interval: 1000
+  # master commit task interval
+  task-commit-interval: 1s
   state-wheel-interval: 5
   # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
   max-cpu-load-avg: -1
   # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
   reserved-memory: 0.3
   # failover interval, the unit is minute
-  failover-interval: 10
+  failover-interval: 10m
   # kill yarn jon when failover taskInstance, default true
   kill-yarn-job-when-task-failover: true
 
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java
index d5a1a79cf3..64bc291152 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/BlockingTaskTest.java
@@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.runner.task.TaskAction;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
@@ -77,7 +78,7 @@ public class BlockingTaskTest {
         config = new MasterConfig();
         Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
         config.setTaskCommitRetryTimes(3);
-        config.setTaskCommitInterval(1000);
+        config.setTaskCommitInterval(Duration.ofSeconds(1));
 
         // mock process service
         processService = Mockito.mock(ProcessService.class);
@@ -122,7 +123,7 @@ public class BlockingTaskTest {
         Mockito.when(processService
             .submitTaskWithRetry(Mockito.any(ProcessInstance.class)
                 , Mockito.any(TaskInstance.class)
-                , Mockito.any(Integer.class), Mockito.any(Integer.class)))
+                , Mockito.any(Integer.class), Mockito.any(Long.class)))
             .thenReturn(taskInstance);
         return taskInstance;
     }
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
index c4d90718b6..7d01c44746 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/ConditionsTaskTest.java
@@ -34,6 +34,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
+import java.time.Duration;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -66,7 +67,7 @@ public class ConditionsTaskTest {
         MasterConfig config = new MasterConfig();
         Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
         config.setTaskCommitRetryTimes(3);
-        config.setTaskCommitInterval(1000);
+        config.setTaskCommitInterval(Duration.ofSeconds(1));
 
         processService = Mockito.mock(ProcessService.class);
         Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
index 8172fd44a0..50b03c86e2 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/DependentTaskTest.java
@@ -35,6 +35,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
+import java.time.Duration;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -85,7 +86,7 @@ public class DependentTaskTest {
 
         MasterConfig config = new MasterConfig();
         config.setTaskCommitRetryTimes(3);
-        config.setTaskCommitInterval(1000);
+        config.setTaskCommitInterval(Duration.ofSeconds(1));
         Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
 
         processService = Mockito.mock(ProcessService.class);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
index 200dec0b3d..fa51a2d6be 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SubProcessTaskTest.java
@@ -44,6 +44,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
 import org.powermock.modules.junit4.PowerMockRunner;
 import org.springframework.context.ApplicationContext;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -69,7 +70,7 @@ public class SubProcessTaskTest {
         MasterConfig config = new MasterConfig();
         Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
         config.setTaskCommitRetryTimes(3);
-        config.setTaskCommitInterval(1000);
+        config.setTaskCommitInterval(Duration.ofSeconds(1));
 
         PowerMockito.mockStatic(Stopper.class);
         PowerMockito.when(Stopper.isRunning()).thenReturn(true);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
index 824726bb6f..6f94a22e2f 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/SwitchTaskTest.java
@@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -59,7 +60,7 @@ public class SwitchTaskTest {
         MasterConfig config = new MasterConfig();
         Mockito.when(applicationContext.getBean(MasterConfig.class)).thenReturn(config);
         config.setTaskCommitRetryTimes(3);
-        config.setTaskCommitInterval(1000);
+        config.setTaskCommitInterval(Duration.ofSeconds(1));
 
         processService = Mockito.mock(ProcessService.class);
         Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index d6131174b3..588fb59422 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -103,7 +103,7 @@ public interface ProcessService {
 
     void setSubProcessParam(ProcessInstance subProcessInstance);
 
-    TaskInstance submitTaskWithRetry(ProcessInstance processInstance, TaskInstance taskInstance, int commitRetryTimes, int commitInterval);
+    TaskInstance submitTaskWithRetry(ProcessInstance processInstance, TaskInstance taskInstance, int commitRetryTimes, long commitInterval);
 
     @Transactional(rollbackFor = Exception.class)
     TaskInstance submitTask(ProcessInstance processInstance, TaskInstance taskInstance);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 7021f4aeef..965ab89801 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -1233,7 +1233,7 @@ public class ProcessServiceImpl implements ProcessService {
      * retry submit task to db
      */
     @Override
-    public TaskInstance submitTaskWithRetry(ProcessInstance processInstance, TaskInstance taskInstance, int commitRetryTimes, int commitInterval) {
+    public TaskInstance submitTaskWithRetry(ProcessInstance processInstance, TaskInstance taskInstance, int commitRetryTimes, long commitInterval) {
         int retryTimes = 1;
         TaskInstance task = null;
         while (retryTimes <= commitRetryTimes) {
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index 7be661de4e..cf5ce5ea88 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -114,19 +114,19 @@ master:
   dispatch-task-number: 3
   # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
   host-selector: lower_weight
-  # master heartbeat interval, the unit is second
-  heartbeat-interval: 10
+  # master heartbeat interval
+  heartbeat-interval: 10s
   # master commit task retry times
   task-commit-retry-times: 5
-  # master commit task interval, the unit is millisecond
-  task-commit-interval: 1000
+  # master commit task interval
+  task-commit-interval: 1s
   state-wheel-interval: 5
   # master max cpuload avg, only higher than the system cpu load average, master server can schedule. default value -1: the number of cpu cores * 2
   max-cpu-load-avg: -1
   # master reserved memory, only lower than system available memory, master server can schedule. default value 0.3, the unit is G
   reserved-memory: 0.3
-  # failover interval, the unit is minute
-  failover-interval: 10
+  # failover interval
+  failover-interval: 10m
   # kill yarn jon when failover taskInstance, default true
   kill-yarn-job-when-task-failover: true
 
@@ -135,8 +135,8 @@ worker:
   listen-port: 1234
   # worker execute thread number to limit task instances in parallel
   exec-threads: 10
-  # worker heartbeat interval, the unit is second
-  heartbeat-interval: 10
+  # worker heartbeat interval
+  heartbeat-interval: 10s
   # worker host weight to dispatch tasks, default value 100
   host-weight: 100
   # worker tenant auto create
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
index 6edff09f97..a641b28e1b 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java
@@ -17,104 +17,53 @@
 
 package org.apache.dolphinscheduler.server.worker.config;
 
+import java.time.Duration;
 import java.util.Set;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.validation.Errors;
+import org.springframework.validation.Validator;
+import org.springframework.validation.annotation.Validated;
 
-@Configuration
-@EnableConfigurationProperties
-@ConfigurationProperties("worker")
-public class WorkerConfig {
-    private int listenPort;
-    private int execThreads;
-    private int heartbeatInterval;
-    private int hostWeight;
-    private boolean tenantAutoCreate;
-    private int maxCpuLoadAvg;
-    private double reservedMemory;
-    private Set<String> groups;
-    private String alertListenHost;
-    private int alertListenPort;
-
-    public int getListenPort() {
-        return listenPort;
-    }
-
-    public void setListenPort(int listenPort) {
-        this.listenPort = listenPort;
-    }
-
-    public int getExecThreads() {
-        return execThreads;
-    }
-
-    public void setExecThreads(int execThreads) {
-        this.execThreads = execThreads;
-    }
-
-    public int getHeartbeatInterval() {
-        return heartbeatInterval;
-    }
-
-    public void setHeartbeatInterval(int heartbeatInterval) {
-        this.heartbeatInterval = heartbeatInterval;
-    }
-
-    public int getHostWeight() {
-        return hostWeight;
-    }
-
-    public void setHostWeight(int hostWeight) {
-        this.hostWeight = hostWeight;
-    }
-
-    public boolean isTenantAutoCreate() {
-        return tenantAutoCreate;
-    }
+import com.google.common.collect.Sets;
 
-    public void setTenantAutoCreate(boolean tenantAutoCreate) {
-        this.tenantAutoCreate = tenantAutoCreate;
-    }
-
-    public int getMaxCpuLoadAvg() {
-        return maxCpuLoadAvg > 0 ? maxCpuLoadAvg : Runtime.getRuntime().availableProcessors() * 2;
-    }
-
-    public void setMaxCpuLoadAvg(int maxCpuLoadAvg) {
-        this.maxCpuLoadAvg = maxCpuLoadAvg;
-    }
+import lombok.Data;
 
-    public double getReservedMemory() {
-        return reservedMemory;
-    }
-
-    public void setReservedMemory(double reservedMemory) {
-        this.reservedMemory = reservedMemory;
-    }
-
-    public Set<String> getGroups() {
-        return groups;
-    }
-
-    public void setGroups(Set<String> groups) {
-        this.groups = groups;
-    }
-
-    public String getAlertListenHost() {
-        return alertListenHost;
-    }
-
-    public void setAlertListenHost(String alertListenHost) {
-        this.alertListenHost = alertListenHost;
-    }
-
-    public int getAlertListenPort() {
-        return alertListenPort;
-    }
+@Data
+@Validated
+@Configuration
+@ConfigurationProperties(prefix = "worker")
+public class WorkerConfig implements Validator {
+    private int listenPort = 1234;
+    private int execThreads = 10;
+    private Duration heartbeatInterval = Duration.ofSeconds(10);
+    private int hostWeight = 100;
+    private boolean tenantAutoCreate = true;
+    private boolean tenantDistributedUser = false;
+    private int maxCpuLoadAvg = -1;
+    private double reservedMemory = 0.3;
+    private Set<String> groups = Sets.newHashSet("default");
+    private String alertListenHost = "localhost";
+    private int alertListenPort = 50052;
+
+    @Override
+    public boolean supports(Class<?> clazz) {
+        return WorkerConfig.class.isAssignableFrom(clazz);
+    }
+
+    @Override
+    public void validate(Object target, Errors errors) {
+        WorkerConfig workerConfig = (WorkerConfig) target;
+        if (workerConfig.getExecThreads() <= 0) {
+            errors.rejectValue("exec-threads", null, "should be a positive value");
+        }
+        if (workerConfig.getHeartbeatInterval().toMillis() <= 0) {
+            errors.rejectValue("heartbeat-interval", null, "shoule be a valid duration");
+        }
+        if (workerConfig.getMaxCpuLoadAvg() <= 0) {
+            workerConfig.setMaxCpuLoadAvg(Runtime.getRuntime().availableProcessors() * 2);
+        }
 
-    public void setAlertListenPort(final int alertListenPort) {
-        this.alertListenPort = alertListenPort;
     }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
index e218b0a4aa..b33b3ef0d7 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
@@ -99,7 +99,7 @@ public class WorkerRegistryClient {
     public void registry() {
         String address = NetUtils.getAddr(workerConfig.getListenPort());
         Set<String> workerZkPaths = getWorkerZkPaths();
-        int workerHeartbeatInterval = workerConfig.getHeartbeatInterval();
+        long workerHeartbeatInterval = workerConfig.getHeartbeatInterval().getSeconds();
 
         HeartBeatTask heartBeatTask = new HeartBeatTask(startupTime,
                 workerConfig.getMaxCpuLoadAvg(),
diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml
index 397e533db9..d764f9d25b 100644
--- a/dolphinscheduler-worker/src/main/resources/application.yaml
+++ b/dolphinscheduler-worker/src/main/resources/application.yaml
@@ -58,8 +58,8 @@ worker:
   listen-port: 1234
   # worker execute thread number to limit task instances in parallel
   exec-threads: 100
-  # worker heartbeat interval, the unit is second
-  heartbeat-interval: 10
+  # worker heartbeat interval
+  heartbeat-interval: 10s
   # worker host weight to dispatch tasks, default value 100
   host-weight: 100
   # worker tenant auto create
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
index 56af82d9d8..4bd2161e83 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
 import org.apache.dolphinscheduler.service.registry.RegistryClient;
 
+import java.time.Duration;
 import java.util.Set;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -86,15 +87,15 @@ public class WorkerRegistryClientTest {
     @Test
     public void testRegistry() {
         workerRegistryClient.initWorkRegistry();
-    
+
         given(workerManagerThread.getThreadPoolQueueSize()).willReturn(1);
-    
+
         given(registryClient.checkNodeExists(Mockito.anyString(), Mockito.any(NodeType.class))).willReturn(true);
-    
-        given(workerConfig.getHeartbeatInterval()).willReturn(1);
-    
+
+        given(workerConfig.getHeartbeatInterval()).willReturn(Duration.ofSeconds(1));
+
         workerRegistryClient.registry();
-    
+
         Mockito.verify(registryClient, Mockito.times(1)).handleDeadServer(Mockito.anyCollection(), Mockito.any(NodeType.class), Mockito.anyString());
     }
 


[dolphinscheduler] 12/29: Fix compile error

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit c488a9f82811babe12307b6c37b9433b50a6a0c5
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jul 19 11:35:56 2022 +0800

    Fix compile error
---
 .../worker/processor/TaskExecuteProcessor.java     | 134 ++++++++++-----------
 .../server/worker/runner/WorkerManagerThread.java  |   1 -
 2 files changed, 61 insertions(+), 74 deletions(-)

diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
index b355c6702b..484acf418e 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
@@ -42,8 +42,6 @@ import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
 import org.apache.dolphinscheduler.service.task.TaskPluginManager;
 
-import org.apache.commons.lang.SystemUtils;
-
 import java.util.Date;
 
 import org.slf4j.Logger;
@@ -133,84 +131,74 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
             taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext));
 
             if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) {
-                boolean osUserExistFlag;
-                //if Using distributed is true and Currently supported systems are linux,Should not let it automatically
-                //create tenants,so TenantAutoCreate has no effect
-                if (workerConfig.isTenantDistributedUser() && SystemUtils.IS_OS_LINUX) {
-                    //use the id command to judge in linux
-                    osUserExistFlag = OSUtils.existTenantCodeInLinux(taskExecutionContext.getTenantCode());
-                } else if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) {
-                    // if not exists this user, then create
+                if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) {
                     OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode());
-                    osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode());
-                } else {
-                    osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode());
-                }
-                if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) {
-                    if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) {
-                        OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode());
-                    }
-
-                    // check if the OS user exists
-                    if (!OSUtils.getUserList().contains(taskExecutionContext.getTenantCode())) {
-                        logger.error("tenantCode: {} does not exist, taskInstanceId: {}",
-                                     taskExecutionContext.getTenantCode(),
-                                     taskExecutionContext.getTaskInstanceId());
-                        TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-                        taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
-                        taskExecutionContext.setEndTime(new Date());
-                        taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
-                        return;
-                    }
-
-                    // local execute path
-                    String execLocalPath = getExecLocalPath(taskExecutionContext);
-                    logger.info("task instance local execute path : {}", execLocalPath);
-                    taskExecutionContext.setExecutePath(execLocalPath);
-
-                    try {
-                        FileUtils.createWorkDirIfAbsent(execLocalPath);
-                    } catch (Throwable ex) {
-                        logger.error("create execLocalPath fail, path: {}, taskInstanceId: {}",
-                                     execLocalPath,
-                                     taskExecutionContext.getTaskInstanceId());
-                        logger.error("create executeLocalPath fail", ex);
-                        TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-                        taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
-                        taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
-                        return;
-                    }
                 }
 
-                taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
-                                                     new NettyRemoteChannel(channel, command.getOpaque()));
-
-                // delay task process
-                long remainTime = DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(),
-                                                          taskExecutionContext.getDelayTime() * 60L);
-                if (remainTime > 0) {
-                    logger.info("delay the execution of task instance {}, delay time: {} s",
-                                taskExecutionContext.getTaskInstanceId(),
-                                remainTime);
-                    taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION);
-                    taskExecutionContext.setStartTime(null);
-                    taskCallbackService.sendTaskExecuteDelayCommand(taskExecutionContext);
+                // check if the OS user exists
+                if (!OSUtils.getUserList().contains(taskExecutionContext.getTenantCode())) {
+                    logger.error("tenantCode: {} does not exist, taskInstanceId: {}",
+                                 taskExecutionContext.getTenantCode(),
+                                 taskExecutionContext.getTaskInstanceId());
+                    TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+                    taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
+                    taskExecutionContext.setEndTime(new Date());
+                    taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                    return;
                 }
 
-                // submit task to manager
-                boolean offer = workerManager.offer(new TaskExecuteThread(taskExecutionContext,
-                                                                          taskCallbackService,
-                                                                          alertClientService,
-                                                                          taskPluginManager));
-                if (!offer) {
-                    logger.error("submit task to manager error, queue is full, queue size is {}, taskInstanceId: {}",
-                                 workerManager.getDelayQueueSize(),
+                // local execute path
+                String execLocalPath = getExecLocalPath(taskExecutionContext);
+                logger.info("task instance local execute path : {}", execLocalPath);
+                taskExecutionContext.setExecutePath(execLocalPath);
+
+                try {
+                    FileUtils.createWorkDirIfAbsent(execLocalPath);
+                } catch (Throwable ex) {
+                    logger.error("create execLocalPath fail, path: {}, taskInstanceId: {}",
+                                 execLocalPath,
                                  taskExecutionContext.getTaskInstanceId());
+                    logger.error("create executeLocalPath fail", ex);
+                    TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
                     taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
                     taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                    return;
                 }
             }
 
+            taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
+                                                 new NettyRemoteChannel(channel, command.getOpaque()));
+
+            // delay task process
+            long remainTime = DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(),
+                                                      taskExecutionContext.getDelayTime() * 60L);
+            if (remainTime > 0) {
+                logger.info("delay the execution of task instance {}, delay time: {} s",
+                            taskExecutionContext.getTaskInstanceId(),
+                            remainTime);
+                taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION);
+                taskExecutionContext.setStartTime(null);
+                taskCallbackService.sendTaskExecuteDelayCommand(taskExecutionContext);
+            }
+
+            // submit task to manager
+            boolean offer = workerManager.offer(new TaskExecuteThread(taskExecutionContext,
+                                                                      taskCallbackService,
+                                                                      alertClientService,
+                                                                      taskPluginManager,
+                                                                      storageOperate));
+            if (!offer) {
+                logger.error("submit task to manager error, queue is full, queue size is {}, taskInstanceId: {}",
+                             workerManager.getDelayQueueSize(),
+                             taskExecutionContext.getTaskInstanceId());
+                taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
+                taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+            }
+        } finally {
+            LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
+        }
+    }
+
     /**
      * get execute local path
      *
@@ -219,9 +207,9 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
      */
     private String getExecLocalPath(TaskExecutionContext taskExecutionContext) {
         return FileUtils.getProcessExecDir(taskExecutionContext.getProjectCode(),
-                taskExecutionContext.getProcessDefineCode(),
-                taskExecutionContext.getProcessDefineVersion(),
-                taskExecutionContext.getProcessInstanceId(),
-                taskExecutionContext.getTaskInstanceId());
+                                           taskExecutionContext.getProcessDefineCode(),
+                                           taskExecutionContext.getProcessDefineVersion(),
+                                           taskExecutionContext.getProcessInstanceId(),
+                                           taskExecutionContext.getTaskInstanceId());
     }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
index ae735f4338..9c826e0d5a 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
@@ -143,7 +143,6 @@ public class WorkerManagerThread implements Runnable {
         while (Stopper.isRunning()) {
             try {
                 taskExecuteThread = workerExecuteQueue.take();
-                taskExecuteThread.setStorageOperate(storageOperate);
                 workerExecService.submit(taskExecuteThread);
             } catch (Exception e) {
                 logger.error("An unexpected interrupt is happened, "


[dolphinscheduler] 05/29: Add lombok

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 1def1729bc14c972a4386c61bcb9a90bcbb0a1d9
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jul 19 11:20:37 2022 +0800

    Add lombok
---
 pom.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/pom.xml b/pom.xml
index 4ea59b0c01..b9572d3a06 100644
--- a/pom.xml
+++ b/pom.xml
@@ -131,6 +131,7 @@
         <hibernate.validator.version>6.2.2.Final</hibernate.validator.version>
         <aws.sdk.version>1.12.160</aws.sdk.version>
         <joda-time.version>2.10.13</joda-time.version>
+        <lombok.version>1.18.20</lombok.version>
         <docker.hub>apache</docker.hub>
         <docker.repo>${project.name}</docker.repo>
         <docker.tag>${project.version}</docker.tag>
@@ -1223,6 +1224,12 @@
             <version>${powermock.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>${lombok.version}</version>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
     <modules>


[dolphinscheduler] 27/29: Kill task when we do master failover to avoid task lost (#10997)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 17f0be5bfbbb17a910ff05e795a582e378d07f63
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Fri Jul 15 18:44:42 2022 +0800

    Kill task when we do master failover to avoid task lost (#10997)
    
    (cherry picked from commit 998e4d74ddf6fce03d65c351b75b91694c761b07)
---
 .../master/service/MasterFailoverService.java      | 21 ++++++++++++++++++++-
 .../server/master/service/FailoverServiceTest.java |  6 +++++-
 .../remote/command/TaskKillRequestCommand.java     | 22 +++++++---------------
 3 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
index 61ba7c3fd6..93bdae2431 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
@@ -28,8 +28,12 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.remote.command.TaskKillRequestCommand;
+import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException;
+import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
 import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
@@ -63,13 +67,17 @@ public class MasterFailoverService {
     private final ProcessService processService;
     private final String localAddress;
 
+    private final NettyExecutorManager nettyExecutorManager;
+
     public MasterFailoverService(@NonNull RegistryClient registryClient,
                                  @NonNull MasterConfig masterConfig,
-                                 @NonNull ProcessService processService) {
+                                 @NonNull ProcessService processService,
+                                 @NonNull NettyExecutorManager nettyExecutorManager) {
         this.registryClient = registryClient;
         this.masterConfig = masterConfig;
         this.processService = processService;
         this.localAddress = NetUtils.getAddr(masterConfig.getListenPort());
+        this.nettyExecutorManager = nettyExecutorManager;
 
     }
 
@@ -221,6 +229,17 @@ public class MasterFailoverService {
                 LOGGER.info("TaskInstance failover begin kill the task related yarn job");
                 ProcessUtils.killYarnJob(taskExecutionContext);
             }
+            // kill worker task, When the master failover and worker failover happened in the same time,
+            // the task may not be failover if we don't set NEED_FAULT_TOLERANCE.
+            // This can be improved if we can load all task when cache a workflowInstance in memory
+            try {
+                TaskKillRequestCommand killCommand = new TaskKillRequestCommand(taskInstance.getId());
+                Host workerHost = Host.of(taskInstance.getHost());
+                nettyExecutorManager.doExecute(workerHost, killCommand.convert2Command());
+                LOGGER.info("Failover task success, has killed the task in worker: {}", taskInstance.getHost());
+            } catch (ExecuteException e) {
+                LOGGER.error("Kill task failed", e);
+            }
         } else {
             LOGGER.info("The failover taskInstance is a master task");
         }
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
index 44e5a382f6..ab71cd75cd 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
@@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
 import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
@@ -82,6 +83,9 @@ public class FailoverServiceTest {
     @Mock
     private ProcessInstanceExecCacheManager cacheManager;
 
+    @Mock
+    private NettyExecutorManager nettyExecutorManager;
+
     private static int masterPort = 5678;
     private static int workerPort = 1234;
 
@@ -100,7 +104,7 @@ public class FailoverServiceTest {
 
         given(masterConfig.getListenPort()).willReturn(masterPort);
         MasterFailoverService masterFailoverService =
-            new MasterFailoverService(registryClient, masterConfig, processService);
+            new MasterFailoverService(registryClient, masterConfig, processService, nettyExecutorManager);
         WorkerFailoverService workerFailoverService = new WorkerFailoverService(registryClient,
             masterConfig,
             processService,
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java
index 155b31785e..7dc03d778c 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskKillRequestCommand.java
@@ -21,9 +21,16 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
 
 import java.io.Serializable;
 
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
 /**
  *  kill task request command
  */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
 public class TaskKillRequestCommand implements Serializable {
 
     /**
@@ -31,14 +38,6 @@ public class TaskKillRequestCommand implements Serializable {
      */
     private int taskInstanceId;
 
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-    }
-
     /**
      *  package request command
      *
@@ -51,11 +50,4 @@ public class TaskKillRequestCommand implements Serializable {
         command.setBody(body);
         return command;
     }
-
-    @Override
-    public String toString() {
-        return "TaskKillRequestCommand{"
-                + "taskInstanceId=" + taskInstanceId
-                + '}';
-    }
 }


[dolphinscheduler] 25/29: Fix taskInstance's host is not worker nettyServer address (#10926)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 812d7a8f267076df595c25765274b8b7853b42fc
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Wed Jul 13 20:46:33 2022 +0800

    Fix taskInstance's host is not worker nettyServer address (#10926)
    
    * Fix taskInstance's host is not worker nettyServer address
    
    * Remove unnecessary mock
    
    (cherry picked from commit df0416c1937b8a3a774c6070b958769c48a6a5e0)
---
 .../server/master/processor/TaskExecuteResponseProcessor.java  |  4 +++-
 .../server/master/processor/TaskExecuteRunningProcessor.java   |  4 +++-
 .../server/master/processor/queue/TaskEvent.java               |  9 ++++-----
 .../server/master/processor/queue/TaskResponseServiceTest.java | 10 ++++++----
 4 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
index 7f438e1d9c..775aded34a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
@@ -60,7 +60,9 @@ public class TaskExecuteResponseProcessor implements NettyRequestProcessor {
 
         TaskExecuteResultCommand taskExecuteResultMessage = JSONUtils.parseObject(command.getBody(),
                                                                                   TaskExecuteResultCommand.class);
-        TaskEvent taskResultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, channel);
+        TaskEvent taskResultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage,
+                                                             channel,
+                                                             taskExecuteResultMessage.getMessageSenderAddress());
         try {
             LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskResultEvent.getProcessInstanceId(),
                                                         taskResultEvent.getTaskInstanceId());
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java
index 96ff1ca405..47a66a71f0 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteRunningProcessor.java
@@ -57,7 +57,9 @@ public class TaskExecuteRunningProcessor implements NettyRequestProcessor {
         TaskExecuteRunningCommand taskExecuteRunningMessage = JSONUtils.parseObject(command.getBody(), TaskExecuteRunningCommand.class);
         logger.info("taskExecuteRunningCommand: {}", taskExecuteRunningMessage);
 
-        TaskEvent taskEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, channel);
+        TaskEvent taskEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage,
+                                                        channel,
+                                                        taskExecuteRunningMessage.getMessageSenderAddress());
         taskEventService.addEvent(taskEvent);
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
index 7db0bc6696..797ccc8a7c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteResultCommand;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningCommand;
-import org.apache.dolphinscheduler.remote.utils.ChannelUtils;
 
 import java.util.Date;
 
@@ -105,7 +104,7 @@ public class TaskEvent {
         return event;
     }
 
-    public static TaskEvent newRunningEvent(TaskExecuteRunningCommand command, Channel channel) {
+    public static TaskEvent newRunningEvent(TaskExecuteRunningCommand command, Channel channel, String workerAddress) {
         TaskEvent event = new TaskEvent();
         event.setProcessInstanceId(command.getProcessInstanceId());
         event.setTaskInstanceId(command.getTaskInstanceId());
@@ -114,12 +113,12 @@ public class TaskEvent {
         event.setExecutePath(command.getExecutePath());
         event.setLogPath(command.getLogPath());
         event.setChannel(channel);
-        event.setWorkerAddress(ChannelUtils.toAddress(channel).getAddress());
+        event.setWorkerAddress(workerAddress);
         event.setEvent(TaskEventType.RUNNING);
         return event;
     }
 
-    public static TaskEvent newResultEvent(TaskExecuteResultCommand command, Channel channel) {
+    public static TaskEvent newResultEvent(TaskExecuteResultCommand command, Channel channel, String workerAddress) {
         TaskEvent event = new TaskEvent();
         event.setProcessInstanceId(command.getProcessInstanceId());
         event.setTaskInstanceId(command.getTaskInstanceId());
@@ -132,7 +131,7 @@ public class TaskEvent {
         event.setAppIds(command.getAppIds());
         event.setVarPool(command.getVarPool());
         event.setChannel(channel);
-        event.setWorkerAddress(ChannelUtils.toAddress(channel).getAddress());
+        event.setWorkerAddress(workerAddress);
         event.setEvent(TaskEventType.RESULT);
         return event;
     }
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
index 3854ad77b0..6f4fe9ceb7 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskResponseServiceTest.java
@@ -75,8 +75,6 @@ public class TaskResponseServiceTest {
     public void before() {
         taskEventService.start();
 
-        Mockito.when(channel.remoteAddress()).thenReturn(InetSocketAddress.createUnresolved("127.0.0.1", 1234));
-
         TaskExecuteRunningCommand taskExecuteRunningMessage = new TaskExecuteRunningCommand("127.0.0.1:5678",
                                                                                             "127.0.0.1:1234",
                                                                                             System.currentTimeMillis());
@@ -88,7 +86,9 @@ public class TaskResponseServiceTest {
         taskExecuteRunningMessage.setHost("127.*.*.*");
         taskExecuteRunningMessage.setStartTime(new Date());
 
-        ackEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage, channel);
+        ackEvent = TaskEvent.newRunningEvent(taskExecuteRunningMessage,
+                                             channel,
+                                             taskExecuteRunningMessage.getMessageSenderAddress());
 
         TaskExecuteResultCommand taskExecuteResultMessage = new TaskExecuteResultCommand(NetUtils.getAddr(1234),
                                                                                          NetUtils.getAddr(5678),
@@ -100,7 +100,9 @@ public class TaskResponseServiceTest {
         taskExecuteResultMessage.setVarPool("varPol");
         taskExecuteResultMessage.setAppIds("ids");
         taskExecuteResultMessage.setProcessId(1);
-        resultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage, channel);
+        resultEvent = TaskEvent.newResultEvent(taskExecuteResultMessage,
+                                               channel,
+                                               taskExecuteResultMessage.getMessageSenderAddress());
 
         taskInstance = new TaskInstance();
         taskInstance.setId(22);


[dolphinscheduler] 22/29: [Fix-10854] Fix database restart may lost task instance status (#10866)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit b259deb196c6011c228f88659250abede63e852d
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Mon Jul 11 09:57:00 2022 +0800

    [Fix-10854] Fix database restart may lost task instance status (#10866)
    
    * Fix database update error doesn't rollback the task instance status
    
    * Fix database error may cause workflow dead with running status
    
    (cherry picked from commit f639a2eed464abc0045d970b2e61397b4c488772)
---
 .../common/enums/StateEventType.java               |   2 +-
 .../enums/{Event.java => TaskEventType.java}       |   2 +-
 .../dao/utils/TaskInstanceUtils.java               |  79 +++++++
 .../dao/utils/TaskInstanceUtilsTest.java           |  43 ++++
 .../server/master/MasterServer.java                |  10 +-
 .../server/master/event/TaskDelayEventHandler.java | 120 ++++++++++
 .../master/event/TaskDispatchEventHandler.java     |  88 ++++++++
 .../server/master/event/TaskEventHandleError.java  |  17 +-
 .../master/event/TaskEventHandleException.java     |  17 +-
 .../server/master/event/TaskEventHandler.java      |  22 +-
 .../event/TaskRejectByWorkerEventHandler.java      |  77 +++++++
 .../master/event/TaskResultEventHandler.java       | 117 ++++++++++
 .../master/event/TaskRunningEventHandler.java      | 118 ++++++++++
 .../server/master/event/WorkflowEvent.java         |  19 +-
 .../master/event/WorkflowEventHandleError.java     |  17 +-
 .../master/event/WorkflowEventHandleException.java |  17 +-
 .../server/master/event/WorkflowEventHandler.java  |  19 +-
 .../server/master/event/WorkflowEventQueue.java    |  48 ++++
 .../server/master/event/WorkflowEventType.java     |  11 +-
 .../master/event/WorkflowStartEventHandler.java    |  86 ++++++++
 .../master/event/WorkflowStateEventHandler.java    |   9 +-
 .../master/processor/StateEventProcessor.java      |  11 +-
 .../processor/queue/StateEventResponseService.java |   6 +-
 .../server/master/processor/queue/TaskEvent.java   |  10 +-
 .../processor/queue/TaskExecuteRunnable.java       | 189 +++-------------
 .../processor/queue/TaskExecuteThreadPool.java     |  31 +--
 ...rService.java => MasterSchedulerBootstrap.java} | 181 +++++----------
 .../server/master/runner/WorkflowEventLooper.java  | 108 +++++++++
 .../master/runner/WorkflowExecuteRunnable.java     | 245 ++++++++++++---------
 .../remote/command/TaskRecallCommand.java          |   0
 .../service/process/ProcessServiceImpl.java        |  16 +-
 .../server/worker/cache/ResponseCache.java         |   4 +-
 .../worker/processor/TaskCallbackService.java      |   6 +-
 .../worker/runner/RetryReportTaskStatusThread.java |  17 +-
 34 files changed, 1246 insertions(+), 516 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java
index 292202501c..bc021e5e08 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEventType.java
@@ -21,7 +21,7 @@ import com.baomidou.mybatisplus.annotation.EnumValue;
 
 public enum StateEventType {
 
-    PROCESS_STATE_CHANGE(0, "process statechange"),
+    PROCESS_STATE_CHANGE(0, "process state change"),
     TASK_STATE_CHANGE(1, "task state change"),
     PROCESS_TIMEOUT(2, "process timeout"),
     TASK_TIMEOUT(3, "task timeout"),
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskEventType.java
similarity index 97%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskEventType.java
index 78b036f037..4ed4a07bf0 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/TaskEventType.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.common.enums;
 
-public enum Event {
+public enum TaskEventType {
     DISPATCH,
     DELAY,
     RUNNING,
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java
new file mode 100644
index 0000000000..fa3bfec0ca
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.utils;
+
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+
+public class TaskInstanceUtils {
+
+    /**
+     * Copy the property of given source {@link TaskInstance} to target.
+     *
+     * @param source Given task instance, copy from.
+     * @param target Given task instance, copy to
+     * @return a soft copy of given task instance.
+     */
+    public static void copyTaskInstance(TaskInstance source, TaskInstance target) {
+        target.setId(source.getId());
+        target.setName(source.getName());
+        target.setTaskType(source.getTaskType());
+        target.setProcessInstanceId(source.getProcessInstanceId());
+        target.setTaskCode(source.getTaskCode());
+        target.setTaskDefinitionVersion(source.getTaskDefinitionVersion());
+        target.setProcessInstanceName(source.getProcessInstanceName());
+        target.setTaskGroupPriority(source.getTaskGroupPriority());
+        target.setState(source.getState());
+        target.setFirstSubmitTime(source.getFirstSubmitTime());
+        target.setSubmitTime(source.getSubmitTime());
+        target.setStartTime(source.getStartTime());
+        target.setEndTime(source.getEndTime());
+        target.setHost(source.getHost());
+        target.setExecutePath(source.getExecutePath());
+        target.setLogPath(source.getLogPath());
+        target.setRetryTimes(source.getRetryTimes());
+        target.setAlertFlag(source.getAlertFlag());
+        target.setProcessInstance(source.getProcessInstance());
+        target.setProcessDefine(source.getProcessDefine());
+        target.setTaskDefine(source.getTaskDefine());
+        target.setPid(source.getPid());
+        target.setAppLink(source.getAppLink());
+        target.setFlag(source.getFlag());
+        target.setDependency(source.getDependency());
+        // todo: we need to cpoy the task params and then copy switchDependency, since the setSwitchDependency rely on task params, this is really a very bad practice.
+        target.setTaskParams(source.getTaskParams());
+        target.setSwitchDependency(source.getSwitchDependency());
+        target.setDuration(source.getDuration());
+        target.setMaxRetryTimes(source.getMaxRetryTimes());
+        target.setRetryInterval(source.getRetryInterval());
+        target.setTaskInstancePriority(source.getTaskInstancePriority());
+        target.setDependentResult(source.getDependentResult());
+        target.setWorkerGroup(source.getWorkerGroup());
+        target.setEnvironmentCode(source.getEnvironmentCode());
+        target.setEnvironmentConfig(source.getEnvironmentConfig());
+        target.setExecutorId(source.getExecutorId());
+        target.setVarPool(source.getVarPool());
+        target.setExecutorName(source.getExecutorName());
+        target.setResources(source.getResources());
+        target.setDelayTime(source.getDelayTime());
+        target.setDryRun(source.getDryRun());
+        target.setTaskGroupId(source.getTaskGroupId());
+        target.setCpuQuota(source.getCpuQuota());
+        target.setMemoryMax(source.getMemoryMax());
+    }
+
+}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java
new file mode 100644
index 0000000000..497c2fb881
--- /dev/null
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtilsTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.utils;
+
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+
+import java.util.Date;
+import java.util.HashMap;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class TaskInstanceUtilsTest {
+
+    @Test
+    void copyTaskInstance() {
+        TaskInstance source = new TaskInstance();
+        source.setId(1);
+        source.setName("source");
+        source.setSubmitTime(new Date());
+        source.setTaskParams(JSONUtils.toJsonString(new HashMap<>()));
+        TaskInstance target = new TaskInstance();
+        TaskInstanceUtils.copyTaskInstance(source, target);
+        Assertions.assertEquals(target.getId(), source.getId());
+        Assertions.assertEquals(target.getName(), source.getName());
+    }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 95409a616b..66b258d5b2 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
 import org.apache.dolphinscheduler.server.master.rpc.MasterRPCServer;
 import org.apache.dolphinscheduler.server.master.runner.EventExecuteService;
 import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread;
-import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerService;
+import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerBootstrap;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.task.TaskPluginManager;
 
@@ -59,7 +59,7 @@ public class MasterServer implements IStoppable {
     private TaskPluginManager taskPluginManager;
 
     @Autowired
-    private MasterSchedulerService masterSchedulerService;
+    private MasterSchedulerBootstrap masterSchedulerBootstrap;
 
     @Autowired
     private Scheduler scheduler;
@@ -94,8 +94,8 @@ public class MasterServer implements IStoppable {
         this.masterRegistryClient.start();
         this.masterRegistryClient.setRegistryStoppable(this);
 
-        this.masterSchedulerService.init();
-        this.masterSchedulerService.start();
+        this.masterSchedulerBootstrap.init();
+        this.masterSchedulerBootstrap.start();
 
         this.eventExecuteService.start();
         this.failoverExecuteThread.start();
@@ -129,7 +129,7 @@ public class MasterServer implements IStoppable {
             // thread sleep 3 seconds for thread quietly stop
             ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
             // close
-            this.masterSchedulerService.close();
+            this.masterSchedulerBootstrap.close();
             this.masterRPCServer.close();
             this.masterRegistryClient.closeRegistry();
             // close spring Context and will invoke method with @PreDestroy annotation to destroy beans.
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java
new file mode 100644
index 0000000000..8e2dcced23
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDelayEventHandler.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+
+import java.util.Optional;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TaskDelayEventHandler implements TaskEventHandler {
+
+    private final Logger logger = LoggerFactory.getLogger(TaskDelayEventHandler.class);
+
+    @Autowired
+    private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
+
+    @Autowired
+    private ProcessService processService;
+
+    @Autowired
+    private WorkflowExecuteThreadPool workflowExecuteThreadPool;
+
+    @Override
+    public void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError {
+        int taskInstanceId = taskEvent.getTaskInstanceId();
+        int processInstanceId = taskEvent.getProcessInstanceId();
+
+        WorkflowExecuteRunnable workflowExecuteRunnable =
+            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        if (workflowExecuteRunnable == null) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError("Cannot find related workflow instance from cache");
+        }
+        Optional<TaskInstance> taskInstanceOptional = workflowExecuteRunnable.getTaskInstance(taskInstanceId);
+        if (!taskInstanceOptional.isPresent()) {
+            sendAckToWorker(taskEvent);
+            return;
+        }
+        TaskInstance taskInstance = taskInstanceOptional.get();
+        if (taskInstance.getState().typeIsFinished()) {
+            logger.warn(
+                "The current task status is: {}, will not handle the running event, this event is delay, will discard this event: {}",
+                taskInstance.getState(),
+                taskEvent);
+            sendAckToWorker(taskEvent);
+            return;
+        }
+
+        TaskInstance oldTaskInstance = new TaskInstance();
+        TaskInstanceUtils.copyTaskInstance(taskInstance, oldTaskInstance);
+        try {
+            taskInstance.setState(taskEvent.getState());
+            taskInstance.setStartTime(taskEvent.getStartTime());
+            taskInstance.setHost(taskEvent.getWorkerAddress());
+            taskInstance.setLogPath(taskEvent.getLogPath());
+            taskInstance.setExecutePath(taskEvent.getExecutePath());
+            taskInstance.setPid(taskEvent.getProcessId());
+            taskInstance.setAppLink(taskEvent.getAppIds());
+            if (!processService.updateTaskInstance(taskInstance)) {
+                throw new TaskEventHandleError("Handle task delay event error, update taskInstance to db failed");
+            }
+            sendAckToWorker(taskEvent);
+        } catch (Exception ex) {
+            TaskInstanceUtils.copyTaskInstance(oldTaskInstance, taskInstance);
+            if (ex instanceof TaskEventHandleError) {
+                throw ex;
+            }
+            throw new TaskEventHandleError("Handle task dispatch event error, update taskInstance to db failed", ex);
+        }
+        StateEvent stateEvent = new StateEvent();
+        stateEvent.setProcessInstanceId(taskEvent.getProcessInstanceId());
+        stateEvent.setTaskInstanceId(taskEvent.getTaskInstanceId());
+        stateEvent.setExecutionStatus(taskEvent.getState());
+        stateEvent.setType(StateEventType.TASK_STATE_CHANGE);
+        workflowExecuteThreadPool.submitStateEvent(stateEvent);
+
+    }
+
+    private void sendAckToWorker(TaskEvent taskEvent) {
+        // If event handle success, send ack to worker to otherwise the worker will retry this event
+        TaskExecuteRunningAckCommand taskExecuteRunningAckCommand =
+            new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        taskEvent.getChannel().writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
+    }
+
+    @Override
+    public TaskEventType getHandleEventType() {
+        return TaskEventType.DELAY;
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDispatchEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDispatchEventHandler.java
new file mode 100644
index 0000000000..9378f0c36e
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskDispatchEventHandler.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TaskDispatchEventHandler implements TaskEventHandler {
+
+    private final Logger logger = LoggerFactory.getLogger(TaskDispatchEventHandler.class);
+
+    @Autowired
+    private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
+
+    @Autowired
+    private ProcessService processService;
+
+    @Override
+    public void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError {
+        int taskInstanceId = taskEvent.getTaskInstanceId();
+        int processInstanceId = taskEvent.getProcessInstanceId();
+
+        WorkflowExecuteRunnable workflowExecuteRunnable =
+            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        if (workflowExecuteRunnable == null) {
+            throw new TaskEventHandleError("Cannot find related workflow instance from cache");
+        }
+        TaskInstance taskInstance = workflowExecuteRunnable.getTaskInstance(taskInstanceId)
+            .orElseThrow(() -> new TaskEventHandleError("Cannot find related taskInstance from cache"));
+        if (taskInstance.getState() != ExecutionStatus.SUBMITTED_SUCCESS) {
+            logger.warn(
+                "The current taskInstance status is not SUBMITTED_SUCCESS, so the dispatch event will be discarded, the current is a delay event, event: {}",
+                taskEvent);
+            return;
+        }
+
+        // todo: we need to just log the old status and rollback these two field, no need to copy all fields
+        TaskInstance oldTaskInstance = new TaskInstance();
+        TaskInstanceUtils.copyTaskInstance(taskInstance, oldTaskInstance);
+        // update the taskInstance status
+        taskInstance.setState(ExecutionStatus.DISPATCH);
+        taskInstance.setHost(taskEvent.getWorkerAddress());
+        try {
+            if (!processService.updateTaskInstance(taskInstance)) {
+                throw new TaskEventHandleError("Handle task dispatch event error, update taskInstance to db failed");
+            }
+        } catch (Exception ex) {
+            // rollback status
+            TaskInstanceUtils.copyTaskInstance(oldTaskInstance, taskInstance);
+            if (ex instanceof TaskEventHandleError) {
+                throw ex;
+            }
+            throw new TaskEventHandleError("Handle task running event error, update taskInstance to db failed", ex);
+        }
+    }
+
+    @Override
+    public TaskEventType getHandleEventType() {
+        return TaskEventType.DISPATCH;
+    }
+}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandleError.java
similarity index 72%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandleError.java
index 78b036f037..deae719f4b 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandleError.java
@@ -15,12 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
-    ;
+public class TaskEventHandleError extends Exception {
+
+    public TaskEventHandleError(String message) {
+        super(message);
+    }
+
+    public TaskEventHandleError(String message, Throwable throwable) {
+        super(message, throwable);
+    }
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandleException.java
similarity index 71%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandleException.java
index 78b036f037..e53a2bbcc3 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandleException.java
@@ -15,12 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
-    ;
+public class TaskEventHandleException extends Exception {
+
+    public TaskEventHandleException(String message) {
+        super(message);
+    }
+
+    public TaskEventHandleException(String message, Throwable throwable) {
+        super(message, throwable);
+    }
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandler.java
similarity index 57%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandler.java
index 78b036f037..5d214cbb5a 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskEventHandler.java
@@ -15,12 +15,20 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
-    ;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
+import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
+
+public interface TaskEventHandler {
+
+    /**
+     * Handle the task event
+     *
+     * @throws TaskEventHandleError     this exception means we will discord this event.
+     * @throws TaskEventHandleException this exception means we need to retry this event
+     */
+    void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError, TaskEventHandleException;
+
+    TaskEventType getHandleEventType();
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java
new file mode 100644
index 0000000000..d09a8364ca
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRejectByWorkerEventHandler.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.remote.command.TaskRecallAckCommand;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TaskRejectByWorkerEventHandler implements TaskEventHandler {
+
+    @Autowired
+    private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
+
+    @Override
+    public void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError {
+        int taskInstanceId = taskEvent.getTaskInstanceId();
+        int processInstanceId = taskEvent.getProcessInstanceId();
+
+        WorkflowExecuteRunnable workflowExecuteRunnable =
+            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        if (workflowExecuteRunnable == null) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError(
+                "Handle task reject event error, cannot find related workflow instance from cache, will discard this event");
+        }
+        TaskInstance taskInstance = workflowExecuteRunnable.getTaskInstance(taskInstanceId).orElseThrow(() -> {
+            sendAckToWorker(taskEvent);
+            return new TaskEventHandleError(
+                "Handle task reject event error, cannot find the taskInstance from cache, will discord this event");
+        });
+        try {
+            // todo: If the worker submit multiple reject response to master, the task instance may be dispatch multiple,
+            // we need to control the worker overload by master rather than worker
+            // if the task resubmit and the worker failover, this task may be dispatch twice?
+            // todo: we need to clear the taskInstance host and rollback the status to submit.
+            workflowExecuteRunnable.resubmit(taskInstance.getTaskCode());
+            sendAckToWorker(taskEvent);
+        } catch (Exception ex) {
+            throw new TaskEventHandleError("Handle task reject event error", ex);
+        }
+
+    }
+
+    public void sendAckToWorker(TaskEvent taskEvent) {
+        TaskRecallAckCommand taskRecallAckCommand =
+            new TaskRecallAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        taskEvent.getChannel().writeAndFlush(taskRecallAckCommand.convert2Command());
+    }
+
+    @Override
+    public TaskEventType getHandleEventType() {
+        return TaskEventType.WORKER_REJECT;
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java
new file mode 100644
index 0000000000..67df03682f
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskResultEventHandler.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
+import org.apache.dolphinscheduler.server.utils.DataQualityResultOperator;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+
+import java.util.Optional;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TaskResultEventHandler implements TaskEventHandler {
+
+    @Autowired
+    private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
+
+    @Autowired
+    private WorkflowExecuteThreadPool workflowExecuteThreadPool;
+
+    @Autowired
+    private DataQualityResultOperator dataQualityResultOperator;
+
+    @Autowired
+    private ProcessService processService;
+
+    @Override
+    public void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError, TaskEventHandleException {
+        int taskInstanceId = taskEvent.getTaskInstanceId();
+        int processInstanceId = taskEvent.getProcessInstanceId();
+
+        WorkflowExecuteRunnable workflowExecuteRunnable =
+            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        if (workflowExecuteRunnable == null) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError(
+                "Handle task result event error, cannot find related workflow instance from cache, will discard this event");
+        }
+        Optional<TaskInstance> taskInstanceOptional = workflowExecuteRunnable.getTaskInstance(taskInstanceId);
+        if (!taskInstanceOptional.isPresent()) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError(
+                "Handle task result event error, cannot find the taskInstance from cache, will discord this event");
+        }
+        TaskInstance taskInstance = taskInstanceOptional.get();
+        if (taskInstance.getState().typeIsFinished()) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError(
+                "Handle task result event error, the task instance is already finished, will discord this event");
+        }
+        dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance);
+
+        TaskInstance oldTaskInstance = new TaskInstance();
+        TaskInstanceUtils.copyTaskInstance(taskInstance, oldTaskInstance);
+        try {
+            taskInstance.setStartTime(taskEvent.getStartTime());
+            taskInstance.setHost(taskEvent.getWorkerAddress());
+            taskInstance.setLogPath(taskEvent.getLogPath());
+            taskInstance.setExecutePath(taskEvent.getExecutePath());
+            taskInstance.setPid(taskEvent.getProcessId());
+            taskInstance.setAppLink(taskEvent.getAppIds());
+            taskInstance.setState(taskEvent.getState());
+            taskInstance.setEndTime(taskEvent.getEndTime());
+            taskInstance.setVarPool(taskEvent.getVarPool());
+            processService.changeOutParam(taskInstance);
+            processService.updateTaskInstance(taskInstance);
+            sendAckToWorker(taskEvent);
+        } catch (Exception ex) {
+            TaskInstanceUtils.copyTaskInstance(oldTaskInstance, taskInstance);
+            throw new TaskEventHandleError("Handle task result event error, save taskInstance to db error", ex);
+        }
+        StateEvent stateEvent = new StateEvent();
+        stateEvent.setProcessInstanceId(taskEvent.getProcessInstanceId());
+        stateEvent.setTaskInstanceId(taskEvent.getTaskInstanceId());
+        stateEvent.setExecutionStatus(taskEvent.getState());
+        stateEvent.setType(StateEventType.TASK_STATE_CHANGE);
+        workflowExecuteThreadPool.submitStateEvent(stateEvent);
+
+    }
+
+    public void sendAckToWorker(TaskEvent taskEvent) {
+        TaskExecuteResponseAckCommand taskExecuteResponseAckCommand =
+            new TaskExecuteResponseAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        taskEvent.getChannel().writeAndFlush(taskExecuteResponseAckCommand.convert2Command());
+    }
+
+    @Override
+    public TaskEventType getHandleEventType() {
+        return TaskEventType.RESULT;
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java
new file mode 100644
index 0000000000..31152973a2
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskRunningEventHandler.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.utils.TaskInstanceUtils;
+import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+
+import java.util.Optional;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class TaskRunningEventHandler implements TaskEventHandler {
+    private final Logger logger = LoggerFactory.getLogger(TaskRunningEventHandler.class);
+
+    @Autowired
+    private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
+
+    @Autowired
+    private WorkflowExecuteThreadPool workflowExecuteThreadPool;
+
+    @Autowired
+    private ProcessService processService;
+
+    @Override
+    public void handleTaskEvent(TaskEvent taskEvent) throws TaskEventHandleError {
+        int taskInstanceId = taskEvent.getTaskInstanceId();
+        int processInstanceId = taskEvent.getProcessInstanceId();
+
+        WorkflowExecuteRunnable workflowExecuteRunnable =
+            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+        if (workflowExecuteRunnable == null) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError(
+                "Handle task running event error, cannot find related workflow instance from cache, will discard this event");
+        }
+        Optional<TaskInstance> taskInstanceOptional = workflowExecuteRunnable.getTaskInstance(taskInstanceId);
+        if (!taskInstanceOptional.isPresent()) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError(
+                "Handle running event error, cannot find the taskInstance from cache, will discord this event");
+        }
+        TaskInstance taskInstance = taskInstanceOptional.get();
+        if (taskInstance.getState().typeIsFinished()) {
+            sendAckToWorker(taskEvent);
+            throw new TaskEventHandleError(
+                "Handle task running event error, this task instance is already finished, this event is delay, will discard this event");
+        }
+
+        TaskInstance oldTaskInstance = new TaskInstance();
+        TaskInstanceUtils.copyTaskInstance(taskInstance, oldTaskInstance);
+        try {
+            taskInstance.setState(taskEvent.getState());
+            taskInstance.setStartTime(taskEvent.getStartTime());
+            taskInstance.setHost(taskEvent.getWorkerAddress());
+            taskInstance.setLogPath(taskEvent.getLogPath());
+            taskInstance.setExecutePath(taskEvent.getExecutePath());
+            taskInstance.setPid(taskEvent.getProcessId());
+            taskInstance.setAppLink(taskEvent.getAppIds());
+            if (!processService.updateTaskInstance(taskInstance)) {
+                throw new TaskEventHandleError("Handle task running event error, update taskInstance to db failed");
+            }
+            sendAckToWorker(taskEvent);
+        } catch (Exception ex) {
+            TaskInstanceUtils.copyTaskInstance(oldTaskInstance, taskInstance);
+            if (ex instanceof TaskEventHandleError) {
+                throw ex;
+            }
+            throw new TaskEventHandleError("Handle task running event error, update taskInstance to db failed", ex);
+        }
+
+        StateEvent stateEvent = new StateEvent();
+        stateEvent.setProcessInstanceId(taskEvent.getProcessInstanceId());
+        stateEvent.setTaskInstanceId(taskEvent.getTaskInstanceId());
+        stateEvent.setExecutionStatus(taskEvent.getState());
+        stateEvent.setType(StateEventType.TASK_STATE_CHANGE);
+        workflowExecuteThreadPool.submitStateEvent(stateEvent);
+    }
+
+    private void sendAckToWorker(TaskEvent taskEvent) {
+        // If event handle success, send ack to worker to otherwise the worker will retry this event
+        TaskExecuteRunningAckCommand taskExecuteRunningAckCommand =
+            new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
+        taskEvent.getChannel().writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
+    }
+
+    @Override
+    public TaskEventType getHandleEventType() {
+        return TaskEventType.RUNNING;
+    }
+}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEvent.java
similarity index 75%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEvent.java
index 78b036f037..5d5bc2eb32 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEvent.java
@@ -15,12 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+@Data
+@AllArgsConstructor
+public class WorkflowEvent {
+
+    private WorkflowEventType workflowEventType;
+
+    private int workflowInstanceId;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
-    ;
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandleError.java
similarity index 71%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandleError.java
index 78b036f037..651f714037 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandleError.java
@@ -15,12 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
-    ;
+public class WorkflowEventHandleError extends Exception {
+
+    public WorkflowEventHandleError(String message) {
+        super(message);
+    }
+
+    public WorkflowEventHandleError(String message, Throwable throwable) {
+        super(message, throwable);
+    }
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandleException.java
similarity index 70%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandleException.java
index 78b036f037..c9b84fd10d 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandleException.java
@@ -15,12 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
-    ;
+public class WorkflowEventHandleException extends Exception {
+
+    public WorkflowEventHandleException(String message) {
+        super(message);
+    }
+
+    public WorkflowEventHandleException(String message, Throwable throwable) {
+        super(message, throwable);
+    }
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandler.java
similarity index 59%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandler.java
index 78b036f037..600d321566 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventHandler.java
@@ -15,12 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
-    ;
+public interface WorkflowEventHandler {
+
+    /**
+     * Handle a workflow event,
+     *
+     * @throws WorkflowEventHandleError     if this exception happen, means the event is broken, need to drop this event.
+     * @throws WorkflowEventHandleException if this exception happen, means we need to retry this event.
+     */
+    void handleWorkflowEvent(WorkflowEvent workflowEvent) throws WorkflowEventHandleError, WorkflowEventHandleException;
+
+    WorkflowEventType getHandleWorkflowEventType();
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventQueue.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventQueue.java
new file mode 100644
index 0000000000..86c8b90cfa
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventQueue.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+@Component
+public class WorkflowEventQueue {
+
+    private final Logger logger = LoggerFactory.getLogger(WorkflowEventQueue.class);
+
+    private static final LinkedBlockingQueue<WorkflowEvent> workflowEventQueue = new LinkedBlockingQueue<>();
+
+    /**
+     * Add a workflow event.
+     */
+    public void addEvent(WorkflowEvent workflowEvent) {
+        workflowEventQueue.add(workflowEvent);
+        logger.info("Added workflow event to workflowEvent queue, event: {}", workflowEvent);
+    }
+
+    /**
+     * Pool the head of the workflow event queue and wait an workflow event.
+     */
+    public WorkflowEvent poolEvent() throws InterruptedException {
+        return workflowEventQueue.take();
+    }
+
+}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventType.java
similarity index 87%
rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventType.java
index 78b036f037..b0f5f09e30 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/Event.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowEventType.java
@@ -15,12 +15,11 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.enums;
+package org.apache.dolphinscheduler.server.master.event;
 
-public enum Event {
-    DISPATCH,
-    DELAY,
-    RUNNING,
-    RESULT,
+public enum WorkflowEventType {
+
+    START_WORKFLOW,
     ;
+
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java
new file mode 100644
index 0000000000..b4d9fc1f85
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStartEventHandler.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.event;
+
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
+import org.apache.dolphinscheduler.server.master.runner.StateWheelExecuteThread;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowSubmitStatue;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class WorkflowStartEventHandler implements WorkflowEventHandler {
+
+    private final Logger logger = LoggerFactory.getLogger(WorkflowStartEventHandler.class);
+
+    @Autowired
+    private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
+
+    @Autowired
+    private StateWheelExecuteThread stateWheelExecuteThread;
+
+    @Autowired
+    private WorkflowExecuteThreadPool workflowExecuteThreadPool;
+
+    @Autowired
+    private WorkflowEventQueue workflowEventQueue;
+
+    @Override
+    public void handleWorkflowEvent(WorkflowEvent workflowEvent) throws WorkflowEventHandleError {
+        logger.info("Handle workflow start event, begin to start a workflow, event: {}", workflowEvent);
+        WorkflowExecuteRunnable workflowExecuteRunnable =
+            processInstanceExecCacheManager.getByProcessInstanceId(workflowEvent.getWorkflowInstanceId());
+        if (workflowExecuteRunnable == null) {
+            throw new WorkflowEventHandleError(
+                "The workflow start event is invalid, cannot find the workflow instance from cache");
+        }
+        ProcessInstance processInstance = workflowExecuteRunnable.getProcessInstance();
+
+        ProcessInstanceMetrics.incProcessInstanceSubmit();
+        CompletableFuture<WorkflowSubmitStatue> workflowSubmitFuture =
+            CompletableFuture.supplyAsync(workflowExecuteRunnable::call, workflowExecuteThreadPool);
+        workflowSubmitFuture.thenAccept(workflowSubmitStatue -> {
+            if (WorkflowSubmitStatue.SUCCESS == workflowSubmitStatue) {
+                // submit failed will resend the event to workflow event queue
+                logger.info("Success submit the workflow instance");
+                if (processInstance.getTimeout() > 0) {
+                    stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
+                }
+            } else {
+                logger.error("Failed to submit the workflow instance, will resend the workflow start event: {}",
+                             workflowEvent);
+                workflowEventQueue.addEvent(new WorkflowEvent(WorkflowEventType.START_WORKFLOW,
+                                                              processInstance.getId()));
+            }
+        });
+    }
+
+    @Override
+    public WorkflowEventType getHandleWorkflowEventType() {
+        return WorkflowEventType.START_WORKFLOW;
+    }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java
index 37d8ceb1da..3abdd879bb 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/WorkflowStateEventHandler.java
@@ -41,15 +41,14 @@ public class WorkflowStateEventHandler implements StateEventHandler {
         ProcessInstance processInstance = workflowExecuteRunnable.getProcessInstance();
         ProcessDefinition processDefinition = processInstance.getProcessDefinition();
 
-        logger.info("process:{} state {} change to {}",
-            processInstance.getId(),
-            processInstance.getState(),
-            stateEvent.getExecutionStatus());
+        logger.info(
+            "Handle workflow instance state event, the current workflow instance state {} will be changed to {}",
+            processInstance.getState(), stateEvent.getExecutionStatus());
 
         if (stateEvent.getExecutionStatus() == ExecutionStatus.STOP) {
             // serial wait execution type needs to wake up the waiting process
             if (processDefinition.getExecutionType().typeIsSerialWait() || processDefinition.getExecutionType()
-                .typeIsSerialPriority()) {
+                                                                                            .typeIsSerialPriority()) {
                 workflowExecuteRunnable.endProcess();
                 return true;
             }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
index f90277d0eb..18afd11ae5 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
@@ -17,7 +17,6 @@
 
 package org.apache.dolphinscheduler.server.master.processor;
 
-import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
@@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
+import org.apache.dolphinscheduler.server.master.event.StateEvent;
 import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService;
 
 import org.slf4j.Logger;
@@ -67,11 +67,12 @@ public class StateEventProcessor implements NettyRequestProcessor {
         stateEvent.setType(type);
 
         try {
-            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(),
+                stateEvent.getTaskInstanceId());
 
-            logger.info("Received state event change command, event: {}", stateEvent);
-            stateEventResponseService.addResponse(stateEvent);
-        }finally {
+            logger.info("Received state change command, event: {}", stateEvent);
+            stateEventResponseService.addStateChangeEvent(stateEvent);
+        } finally {
             LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
         }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
index bdf80bbee9..80c90ee1b7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
@@ -42,9 +42,6 @@ import org.springframework.stereotype.Component;
 
 import io.netty.channel.Channel;
 
-/**
- * task manager
- */
 @Component
 public class StateEventResponseService {
 
@@ -96,7 +93,7 @@ public class StateEventResponseService {
     /**
      * put task to attemptQueue
      */
-    public void addResponse(StateEvent stateEvent) {
+    public void addStateChangeEvent(StateEvent stateEvent) {
         try {
             // check the event is validated
             eventQueue.put(stateEvent);
@@ -154,6 +151,7 @@ public class StateEventResponseService {
             }
 
             WorkflowExecuteRunnable workflowExecuteThread = this.processInstanceExecCacheManager.getByProcessInstanceId(stateEvent.getProcessInstanceId());
+            // We will refresh the task instance status first, if the refresh failed the event will not be removed
             switch (stateEvent.getType()) {
                 case TASK_STATE_CHANGE:
                     workflowExecuteThread.refreshTaskInstance(stateEvent.getTaskInstanceId());
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
index 86cce5170e..2eb60ce8b9 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEvent.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
-import org.apache.dolphinscheduler.common.enums.Event;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningCommand;
@@ -82,7 +82,7 @@ public class TaskEvent {
     /**
      * ack / response
      */
-    private Event event;
+    private TaskEventType event;
 
     /**
      * varPool
@@ -101,7 +101,7 @@ public class TaskEvent {
         event.setProcessInstanceId(processInstanceId);
         event.setTaskInstanceId(taskInstanceId);
         event.setWorkerAddress(workerAddress);
-        event.setEvent(Event.DISPATCH);
+        event.setEvent(TaskEventType.DISPATCH);
         return event;
     }
 
@@ -115,7 +115,7 @@ public class TaskEvent {
         event.setLogPath(command.getLogPath());
         event.setChannel(channel);
         event.setWorkerAddress(ChannelUtils.toAddress(channel).getAddress());
-        event.setEvent(Event.RUNNING);
+        event.setEvent(TaskEventType.RUNNING);
         return event;
     }
 
@@ -133,7 +133,7 @@ public class TaskEvent {
         event.setVarPool(command.getVarPool());
         event.setChannel(channel);
         event.setWorkerAddress(ChannelUtils.toAddress(channel).getAddress());
-        event.setEvent(Event.RESULT);
+        event.setEvent(TaskEventType.RESULT);
         return event;
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
index d57adcbbbc..3c05671a10 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
@@ -17,28 +17,18 @@
 
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
-import org.apache.dolphinscheduler.common.enums.Event;
-import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand;
-import org.apache.dolphinscheduler.remote.command.TaskExecuteRunningAckCommand;
-import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
-import org.apache.dolphinscheduler.server.master.event.StateEvent;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
-import org.apache.dolphinscheduler.server.utils.DataQualityResultOperator;
-import org.apache.dolphinscheduler.service.process.ProcessService;
-
-import java.util.Optional;
+import org.apache.dolphinscheduler.server.master.event.TaskEventHandleError;
+import org.apache.dolphinscheduler.server.master.event.TaskEventHandleException;
+import org.apache.dolphinscheduler.server.master.event.TaskEventHandler;
+
+import java.util.Map;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import io.netty.channel.Channel;
-
 /**
  * task execute thread
  */
@@ -50,34 +40,37 @@ public class TaskExecuteRunnable implements Runnable {
 
     private final ConcurrentLinkedQueue<TaskEvent> events = new ConcurrentLinkedQueue<>();
 
-    private ProcessService processService;
-
-    private WorkflowExecuteThreadPool workflowExecuteThreadPool;
-
-    private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
+    private final Map<TaskEventType, TaskEventHandler> taskEventHandlerMap;
 
-    private DataQualityResultOperator dataQualityResultOperator;
-
-    public TaskExecuteRunnable(int processInstanceId, ProcessService processService, WorkflowExecuteThreadPool workflowExecuteThreadPool,
-                               ProcessInstanceExecCacheManager processInstanceExecCacheManager, DataQualityResultOperator dataQualityResultOperator) {
+    public TaskExecuteRunnable(int processInstanceId, Map<TaskEventType, TaskEventHandler> taskEventHandlerMap) {
         this.processInstanceId = processInstanceId;
-        this.processService = processService;
-        this.workflowExecuteThreadPool = workflowExecuteThreadPool;
-        this.processInstanceExecCacheManager = processInstanceExecCacheManager;
-        this.dataQualityResultOperator = dataQualityResultOperator;
+        this.taskEventHandlerMap = taskEventHandlerMap;
     }
 
     @Override
     public void run() {
         while (!this.events.isEmpty()) {
+            // we handle the task event belongs to one task serial, so if the event comes in wrong order,
             TaskEvent event = this.events.peek();
             try {
                 LoggerUtils.setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), event.getTaskInstanceId());
-                persist(event);
-            } catch (Exception e) {
-                logger.error("persist task event error, event:{}", event, e);
+                logger.info("Handle task event begin: {}", event);
+                taskEventHandlerMap.get(event.getEvent()).handleTaskEvent(event);
+                events.remove(event);
+                logger.info("Handle task event finished: {}", event);
+            } catch (TaskEventHandleException taskEventHandleException) {
+                // we don't need to resubmit this event, since the worker will resubmit this event
+                logger.error("Handle task event failed, this event will be retry later, event: {}", event,
+                    taskEventHandleException);
+            } catch (TaskEventHandleError taskEventHandleError) {
+                logger.error("Handle task event error, this event will be removed, event: {}", event,
+                    taskEventHandleError);
+                events.remove(event);
+            } catch (Exception unknownException) {
+                logger.error("Handle task event error, get a unknown exception, this event will be removed, event: {}",
+                    event, unknownException);
+                events.remove(event);
             } finally {
-                this.events.remove(event);
                 LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
             }
         }
@@ -108,134 +101,4 @@ public class TaskExecuteRunnable implements Runnable {
         return this.events.add(event);
     }
 
-    /**
-     * persist task event
-     *
-     * @param taskEvent taskEvent
-     */
-    private void persist(TaskEvent taskEvent) throws Exception {
-        Event event = taskEvent.getEvent();
-        int taskInstanceId = taskEvent.getTaskInstanceId();
-        int processInstanceId = taskEvent.getProcessInstanceId();
-
-        Optional<TaskInstance> taskInstance;
-        WorkflowExecuteRunnable workflowExecuteRunnable =
-            this.processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
-        if (workflowExecuteRunnable != null && workflowExecuteRunnable.checkTaskInstanceById(taskInstanceId)) {
-            taskInstance = workflowExecuteRunnable.getTaskInstance(taskInstanceId);
-        } else {
-            taskInstance = Optional.ofNullable(processService.findTaskInstanceById(taskInstanceId));
-        }
-
-        boolean needToSendEvent = true;
-        switch (event) {
-            case DISPATCH:
-                needToSendEvent = handleDispatchEvent(taskEvent, taskInstance);
-                // dispatch event do not need to submit state event
-                break;
-            case DELAY:
-            case RUNNING:
-                needToSendEvent = handleRunningEvent(taskEvent, taskInstance);
-                break;
-            case RESULT:
-                handleResultEvent(taskEvent, taskInstance);
-                break;
-            default:
-                throw new IllegalArgumentException("invalid event type : " + event);
-        }
-        if (!needToSendEvent) {
-            logger.info("Handle task event: {} success, there is no need to send a StateEvent", taskEvent);
-            return;
-        }
-
-        StateEvent stateEvent = new StateEvent();
-        stateEvent.setProcessInstanceId(taskEvent.getProcessInstanceId());
-        stateEvent.setTaskInstanceId(taskEvent.getTaskInstanceId());
-        stateEvent.setExecutionStatus(taskEvent.getState());
-        stateEvent.setType(StateEventType.TASK_STATE_CHANGE);
-        workflowExecuteThreadPool.submitStateEvent(stateEvent);
-    }
-
-    /**
-     * handle dispatch event
-     */
-    private boolean handleDispatchEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
-        if (!taskInstanceOptional.isPresent()) {
-            logger.error("taskInstance is null");
-            return false;
-        }
-        TaskInstance taskInstance = taskInstanceOptional.get();
-        if (taskInstance.getState() != ExecutionStatus.SUBMITTED_SUCCESS) {
-            return false;
-        }
-        taskInstance.setState(ExecutionStatus.DISPATCH);
-        taskInstance.setHost(taskEvent.getWorkerAddress());
-        processService.saveTaskInstance(taskInstance);
-        return true;
-    }
-
-    /**
-     * handle running event
-     */
-    private boolean handleRunningEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
-        Channel channel = taskEvent.getChannel();
-        if (taskInstanceOptional.isPresent()) {
-            TaskInstance taskInstance = taskInstanceOptional.get();
-            if (taskInstance.getState().typeIsFinished()) {
-                logger.warn("task is finish, running event is meaningless, taskInstanceId:{}, state:{}",
-                    taskInstance.getId(),
-                    taskInstance.getState());
-                return false;
-            } else {
-                taskInstance.setState(taskEvent.getState());
-                taskInstance.setStartTime(taskEvent.getStartTime());
-                taskInstance.setHost(taskEvent.getWorkerAddress());
-                taskInstance.setLogPath(taskEvent.getLogPath());
-                taskInstance.setExecutePath(taskEvent.getExecutePath());
-                taskInstance.setPid(taskEvent.getProcessId());
-                taskInstance.setAppLink(taskEvent.getAppIds());
-                processService.saveTaskInstance(taskInstance);
-            }
-        }
-        // if taskInstance is null (maybe deleted) or finish. retry will be meaningless . so ack success
-        // send ack to worker
-        TaskExecuteRunningAckCommand taskExecuteRunningAckCommand =
-            new TaskExecuteRunningAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-        channel.writeAndFlush(taskExecuteRunningAckCommand.convert2Command());
-        return true;
-    }
-
-    /**
-     * handle result event
-     */
-    private boolean handleResultEvent(TaskEvent taskEvent, Optional<TaskInstance> taskInstanceOptional) {
-        Channel channel = taskEvent.getChannel();
-        if (taskInstanceOptional.isPresent()) {
-            TaskInstance taskInstance = taskInstanceOptional.get();
-            if (taskInstance.getState().typeIsFinished()) {
-                logger.warn("The current taskInstance has already been finished, taskEvent: {}", taskEvent);
-                return false;
-            }
-
-            dataQualityResultOperator.operateDqExecuteResult(taskEvent, taskInstance);
-
-            taskInstance.setStartTime(taskEvent.getStartTime());
-            taskInstance.setHost(taskEvent.getWorkerAddress());
-            taskInstance.setLogPath(taskEvent.getLogPath());
-            taskInstance.setExecutePath(taskEvent.getExecutePath());
-            taskInstance.setPid(taskEvent.getProcessId());
-            taskInstance.setAppLink(taskEvent.getAppIds());
-            taskInstance.setState(taskEvent.getState());
-            taskInstance.setEndTime(taskEvent.getEndTime());
-            taskInstance.setVarPool(taskEvent.getVarPool());
-            processService.changeOutParam(taskInstance);
-            processService.saveTaskInstance(taskInstance);
-        }
-        // if taskInstance is null (maybe deleted) . retry will be meaningless . so response success
-        TaskExecuteResponseAckCommand taskExecuteResponseAckCommand =
-            new TaskExecuteResponseAckCommand(ExecutionStatus.SUCCESS.getCode(), taskEvent.getTaskInstanceId());
-        channel.writeAndFlush(taskExecuteResponseAckCommand.convert2Command());
-        return true;
-    }
-
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
index ae578d4c03..8d5f2fd3db 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
@@ -17,12 +17,14 @@
 
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
-import org.apache.dolphinscheduler.server.utils.DataQualityResultOperator;
-import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.server.master.event.TaskEventHandler;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 import javax.annotation.PostConstruct;
@@ -48,21 +50,10 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
     @Autowired
     private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
 
-    /**
-     * process service
-     */
-    @Autowired
-    private ProcessService processService;
-
-    /**
-     * data quality result operator
-     */
     @Autowired
-    private DataQualityResultOperator dataQualityResultOperator;
+    private List<TaskEventHandler> taskEventHandlerList;
 
-
-    @Autowired
-    private WorkflowExecuteThreadPool workflowExecuteThreadPool;
+    private Map<TaskEventType, TaskEventHandler> taskEventHandlerMap = new HashMap<>();
 
     /**
      * task event thread map
@@ -75,6 +66,8 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
         this.setThreadNamePrefix("Task-Execute-Thread-");
         this.setMaxPoolSize(masterConfig.getExecThreads());
         this.setCorePoolSize(masterConfig.getExecThreads());
+        taskEventHandlerList.forEach(
+            taskEventHandler -> taskEventHandlerMap.put(taskEventHandler.getHandleEventType(), taskEventHandler));
     }
 
     public void submitTaskEvent(TaskEvent taskEvent) {
@@ -83,11 +76,7 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
             return;
         }
         TaskExecuteRunnable taskExecuteRunnable = taskExecuteThreadMap.computeIfAbsent(taskEvent.getProcessInstanceId(),
-            (processInstanceId) -> new TaskExecuteRunnable(processInstanceId,
-                processService,
-                workflowExecuteThreadPool,
-                processInstanceExecCacheManager,
-                dataQualityResultOperator));
+            (processInstanceId) -> new TaskExecuteRunnable(processInstanceId, taskEventHandlerMap));
         taskExecuteRunnable.addEvent(taskEvent);
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
similarity index 57%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
index 35d3155f43..b4c72209db 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerBootstrap.java
@@ -30,12 +30,12 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue;
 import org.apache.dolphinscheduler.server.master.exception.MasterException;
 import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
 import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
 import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
-import org.apache.dolphinscheduler.service.expand.CuringParamsService;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
 import org.apache.commons.collections4.CollectionUtils;
@@ -43,9 +43,7 @@ import org.apache.commons.collections4.CollectionUtils;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 
 import org.slf4j.Logger;
@@ -53,18 +51,13 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import lombok.NonNull;
-
 /**
  * Master scheduler thread, this thread will consume the commands from database and trigger processInstance executed.
  */
 @Service
-public class MasterSchedulerService extends BaseDaemonThread {
+public class MasterSchedulerBootstrap extends BaseDaemonThread {
 
-    /**
-     * logger of MasterSchedulerService
-     */
-    private static final Logger logger = LoggerFactory.getLogger(MasterSchedulerService.class);
+    private static final Logger logger = LoggerFactory.getLogger(MasterSchedulerBootstrap.class);
 
     @Autowired
     private ProcessService processService;
@@ -83,25 +76,21 @@ public class MasterSchedulerService extends BaseDaemonThread {
      */
     private ThreadPoolExecutor masterPrepareExecService;
 
-    /**
-     * workflow exec service
-     */
-    @Autowired
-    private WorkflowExecuteThreadPool workflowExecuteThreadPool;
-
     @Autowired
     private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
 
     @Autowired
     private StateWheelExecuteThread stateWheelExecuteThread;
 
-    private final LinkedBlockingQueue<ProcessInstance> submitFailedProcessInstances = new LinkedBlockingQueue<>();
+    @Autowired
+    private WorkflowEventQueue workflowEventQueue;
 
-    private Thread failedProcessInstanceResubmitThread;
+    @Autowired
+    private WorkflowEventLooper workflowEventLooper;
 
     private String masterAddress;
 
-    protected MasterSchedulerService() {
+    protected MasterSchedulerBootstrap() {
         super("MasterCommandLoopThread");
     }
 
@@ -111,23 +100,19 @@ public class MasterSchedulerService extends BaseDaemonThread {
     public void init() {
         this.masterPrepareExecService = (ThreadPoolExecutor) ThreadUtils.newDaemonFixedThreadExecutor("MasterPreExecThread", masterConfig.getPreExecThreads());
         this.masterAddress = NetUtils.getAddr(masterConfig.getListenPort());
-        this.failedProcessInstanceResubmitThread = new FailedProcessInstanceResubmitThread(submitFailedProcessInstances);
-        ProcessInstanceMetrics.registerProcessInstanceResubmitGauge(submitFailedProcessInstances::size);
     }
 
     @Override
     public synchronized void start() {
-        logger.info("Master schedule service starting..");
+        logger.info("Master schedule bootstrap starting..");
         super.start();
-        this.failedProcessInstanceResubmitThread.start();
-        logger.info("Master schedule service started...");
+        workflowEventLooper.start();
+        logger.info("Master schedule bootstrap started...");
     }
 
     public void close() {
-        logger.info("Master schedule service stopping...");
-        // these process instances will be failover, so we can safa clear here
-        submitFailedProcessInstances.clear();
-        logger.info("Master schedule service stopped...");
+        logger.info("Master schedule bootstrap stopping...");
+        logger.info("Master schedule bootstrap stopped...");
     }
 
     /**
@@ -137,15 +122,51 @@ public class MasterSchedulerService extends BaseDaemonThread {
     public void run() {
         while (Stopper.isRunning()) {
             try {
-                boolean isOverload = OSUtils.isOverload(masterConfig.getMaxCpuLoadAvg(), masterConfig.getReservedMemory());
+                // todo: if the workflow event queue is much, we need to handle the back pressure
+                boolean isOverload =
+                    OSUtils.isOverload(masterConfig.getMaxCpuLoadAvg(), masterConfig.getReservedMemory());
                 if (isOverload) {
                     MasterServerMetrics.incMasterOverload();
                     Thread.sleep(Constants.SLEEP_TIME_MILLIS);
                     continue;
                 }
-                scheduleWorkflow();
+                List<Command> commands = findCommands();
+                if (CollectionUtils.isEmpty(commands)) {
+                    // indicate that no command ,sleep for 1s
+                    Thread.sleep(Constants.SLEEP_TIME_MILLIS);
+                    continue;
+                }
+
+                List<ProcessInstance> processInstances = command2ProcessInstance(commands);
+                if (CollectionUtils.isEmpty(processInstances)) {
+                    // indicate that the command transform to processInstance error, sleep for 1s
+                    Thread.sleep(Constants.SLEEP_TIME_MILLIS);
+                    continue;
+                }
+                MasterServerMetrics.incMasterConsumeCommand(commands.size());
+
+                processInstances.forEach(processInstance -> {
+                    try {
+                        LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
+                        if (processInstanceExecCacheManager.contains(processInstance.getId())) {
+                            logger.error("The workflow instance is already been cached, this case shouldn't be happened");
+                        }
+                        WorkflowExecuteRunnable workflowRunnable = new WorkflowExecuteRunnable(processInstance,
+                                                                                               processService,
+                                                                                               nettyExecutorManager,
+                                                                                               processAlertManager,
+                                                                                               masterConfig,
+                                                                                               stateWheelExecuteThread,
+                                                                                               curingGlobalParamsService);
+                        processInstanceExecCacheManager.cache(processInstance.getId(), workflowRunnable);
+                        workflowEventQueue.addEvent(new WorkflowEvent(WorkflowEventType.START_WORKFLOW,
+                                                                      processInstance.getId()));
+                    } finally {
+                        LoggerUtils.removeWorkflowInstanceIdMDC();
+                    }
+                });
             } catch (InterruptedException interruptedException) {
-                logger.warn("Master schedule service interrupted, close the loop", interruptedException);
+                logger.warn("Master schedule bootstrap interrupted, close the loop", interruptedException);
                 Thread.currentThread().interrupt();
                 break;
             } catch (Exception e) {
@@ -156,71 +177,9 @@ public class MasterSchedulerService extends BaseDaemonThread {
         }
     }
 
-    /**
-     * Query command from database by slot, and transform to workflow instance, then submit to workflowExecuteThreadPool.
-     */
-    private void scheduleWorkflow() throws InterruptedException, MasterException {
-        List<Command> commands = findCommands();
-        if (CollectionUtils.isEmpty(commands)) {
-            // indicate that no command ,sleep for 1s
-            Thread.sleep(Constants.SLEEP_TIME_MILLIS);
-            return;
-        }
-
-        List<ProcessInstance> processInstances = command2ProcessInstance(commands);
-        if (CollectionUtils.isEmpty(processInstances)) {
-            // indicate that the command transform to processInstance error, sleep for 1s
-            Thread.sleep(Constants.SLEEP_TIME_MILLIS);
-            return;
-        }
-        MasterServerMetrics.incMasterConsumeCommand(commands.size());
-
-        for (ProcessInstance processInstance : processInstances) {
-            submitProcessInstance(processInstance);
-        }
-    }
-
-    private void submitProcessInstance(@NonNull ProcessInstance processInstance) {
-        try {
-            LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
-            logger.info("Master schedule service starting workflow instance");
-            final WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
-                processInstance
-                , processService
-                , nettyExecutorManager
-                , processAlertManager
-                , masterConfig
-                , stateWheelExecuteThread);
-
-            this.processInstanceExecCacheManager.cache(processInstance.getId(), workflowExecuteRunnable);
-            if (processInstance.getTimeout() > 0) {
-                stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
-            }
-            ProcessInstanceMetrics.incProcessInstanceSubmit();
-            CompletableFuture<WorkflowSubmitStatue> workflowSubmitFuture = CompletableFuture.supplyAsync(
-                workflowExecuteRunnable::call, workflowExecuteThreadPool);
-            workflowSubmitFuture.thenAccept(workflowSubmitStatue -> {
-                if (WorkflowSubmitStatue.FAILED == workflowSubmitStatue) {
-                    // submit failed
-                    processInstanceExecCacheManager.removeByProcessInstanceId(processInstance.getId());
-                    stateWheelExecuteThread.removeProcess4TimeoutCheck(processInstance.getId());
-                    submitFailedProcessInstances.add(processInstance);
-                }
-            });
-            logger.info("Master schedule service started workflow instance");
-
-        } catch (Exception ex) {
-            processInstanceExecCacheManager.removeByProcessInstanceId(processInstance.getId());
-            stateWheelExecuteThread.removeProcess4TimeoutCheck(processInstance.getId());
-            logger.info("Master submit workflow to thread pool failed, will remove workflow runnable from cache manager", ex);
-        } finally {
-            LoggerUtils.removeWorkflowInstanceIdMDC();
-        }
-    }
-
     private List<ProcessInstance> command2ProcessInstance(List<Command> commands) throws InterruptedException {
         long commandTransformStartTime = System.currentTimeMillis();
-        logger.info("Master schedule service transforming command to ProcessInstance, commandSize: {}", commands.size());
+        logger.info("Master schedule bootstrap transforming command to ProcessInstance, commandSize: {}", commands.size());
         List<ProcessInstance> processInstances = Collections.synchronizedList(new ArrayList<>(commands.size()));
         CountDownLatch latch = new CountDownLatch(commands.size());
         for (final Command command : commands) {
@@ -250,7 +209,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
 
         // make sure to finish handling command each time before next scan
         latch.await();
-        logger.info("Master schedule service transformed command to ProcessInstance, commandSize: {}, processInstanceSize: {}",
+        logger.info("Master schedule bootstrap transformed command to ProcessInstance, commandSize: {}, processInstanceSize: {}",
             commands.size(), processInstances.size());
         ProcessInstanceMetrics.recordProcessInstanceGenerateTime(System.currentTimeMillis() - commandTransformStartTime);
         return processInstances;
@@ -269,7 +228,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
             int pageSize = masterConfig.getFetchCommandNum();
             final List<Command> result = processService.findCommandPageBySlot(pageSize, pageNumber, masterCount, thisMasterSlot);
             if (CollectionUtils.isNotEmpty(result)) {
-                logger.info("Master schedule service loop command success, command size: {}, current slot: {}, total slot size: {}",
+                logger.info("Master schedule bootstrap loop command success, command size: {}, current slot: {}, total slot size: {}",
                     result.size(), thisMasterSlot, masterCount);
             }
             ProcessInstanceMetrics.recordCommandQueryTime(System.currentTimeMillis() - scheduleStartTime);
@@ -293,34 +252,4 @@ public class MasterSchedulerService extends BaseDaemonThread {
         return state;
     }
 
-    private class FailedProcessInstanceResubmitThread extends Thread {
-
-        private final LinkedBlockingQueue<ProcessInstance> submitFailedProcessInstances;
-
-        public FailedProcessInstanceResubmitThread(LinkedBlockingQueue<ProcessInstance> submitFailedProcessInstances) {
-            logger.info("Starting workflow resubmit thread");
-            this.submitFailedProcessInstances = submitFailedProcessInstances;
-            this.setDaemon(true);
-            this.setName("SubmitFailedProcessInstanceHandleThread");
-            logger.info("Started workflow resubmit thread");
-        }
-
-        @Override
-        public void run() {
-            while (Stopper.isRunning()) {
-                try {
-                    ProcessInstance processInstance = submitFailedProcessInstances.take();
-                    submitProcessInstance(processInstance);
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                    logger.warn("SubmitFailedProcessInstanceHandleThread has been interrupted, will return");
-                    break;
-                }
-
-                // avoid the failed-fast cause CPU higher
-                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
-            }
-        }
-    }
-
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java
new file mode 100644
index 0000000000..ee2e70bfd0
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowEventLooper.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.runner;
+
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
+import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.thread.ThreadUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEvent;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandleError;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandleException;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEventHandler;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEventQueue;
+import org.apache.dolphinscheduler.server.master.event.WorkflowEventType;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.PostConstruct;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class WorkflowEventLooper extends BaseDaemonThread {
+
+    private final Logger logger = LoggerFactory.getLogger(WorkflowEventLooper.class);
+
+    @Autowired
+    private WorkflowEventQueue workflowEventQueue;
+
+    @Autowired
+    private List<WorkflowEventHandler> workflowEventHandlerList;
+
+    private final Map<WorkflowEventType, WorkflowEventHandler> workflowEventHandlerMap = new HashMap<>();
+
+    protected WorkflowEventLooper() {
+        super("WorkflowEventLooper");
+    }
+
+    @PostConstruct
+    public void init() {
+        workflowEventHandlerList.forEach(workflowEventHandler -> workflowEventHandlerMap.put(workflowEventHandler.getHandleWorkflowEventType(),
+                                                                                             workflowEventHandler));
+    }
+
+    @Override
+    public synchronized void start() {
+        logger.info("WorkflowEventLooper thread starting");
+        super.start();
+        logger.info("WorkflowEventLooper thread started");
+    }
+
+    public void run() {
+        WorkflowEvent workflowEvent = null;
+        while (Stopper.isRunning()) {
+            try {
+                workflowEvent = workflowEventQueue.poolEvent();
+                LoggerUtils.setWorkflowInstanceIdMDC(workflowEvent.getWorkflowInstanceId());
+                logger.info("Workflow event looper receive a workflow event: {}, will handle this", workflowEvent);
+                WorkflowEventHandler workflowEventHandler =
+                    workflowEventHandlerMap.get(workflowEvent.getWorkflowEventType());
+                workflowEventHandler.handleWorkflowEvent(workflowEvent);
+            } catch (InterruptedException e) {
+                logger.warn("WorkflowEventLooper thread is interrupted, will close this loop", e);
+                Thread.currentThread().interrupt();
+                break;
+            } catch (WorkflowEventHandleException workflowEventHandleException) {
+                logger.error("Handle workflow event failed, will add this event to event queue again, event: {}",
+                    workflowEvent, workflowEventHandleException);
+                workflowEventQueue.addEvent(workflowEvent);
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
+            } catch (WorkflowEventHandleError workflowEventHandleError) {
+                logger.error("Handle workflow event error, will drop this event, event: {}",
+                             workflowEvent,
+                             workflowEventHandleError);
+            } catch (Exception unknownException) {
+                logger.error(
+                    "Handle workflow event failed, get a unknown exception, will add this event to event queue again, event: {}",
+                    workflowEvent, unknownException);
+                workflowEventQueue.addEvent(workflowEvent);
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
+            }
+        }
+    }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 2763d99932..a433981aca 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -118,39 +118,18 @@ import lombok.NonNull;
  */
 public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
 
-    /**
-     * logger of WorkflowExecuteThread
-     */
     private static final Logger logger = LoggerFactory.getLogger(WorkflowExecuteRunnable.class);
 
-    /**
-     * process service
-     */
     private final ProcessService processService;
 
-    /**
-     * alert manager
-     */
     private final ProcessAlertManager processAlertManager;
 
-    /**
-     * netty executor manager
-     */
     private final NettyExecutorManager nettyExecutorManager;
 
-    /**
-     * process instance
-     */
     private final ProcessInstance processInstance;
 
-    /**
-     * process definition
-     */
     private ProcessDefinition processDefinition;
 
-    /**
-     * the object of DAG
-     */
     private DAG<String, TaskNode, TaskNodeRelation> dag;
 
     /**
@@ -158,10 +137,8 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      */
     private String key;
 
-    /**
-     * start flag, true: start nodes submit completely
-     */
-    private volatile boolean isStart = false;
+
+    private WorkflowRunnableStatus workflowRunnableStatus = WorkflowRunnableStatus.CREATED;
 
     /**
      * submit failure nodes
@@ -223,7 +200,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     private final ConcurrentLinkedQueue<StateEvent> stateEvents = new ConcurrentLinkedQueue<>();
 
     /**
-     * ready to submit task queue
+     * The StandBy task list, will be executed, need to know, the taskInstance in this queue may doesn't have id.
      */
     private final PeerTaskInstancePriorityQueue readyToSubmitTaskQueue = new PeerTaskInstancePriorityQueue();
 
@@ -233,9 +210,6 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      */
     private final Map<Long, TaskInstance> waitToRetryTaskInstanceMap = new ConcurrentHashMap<>();
 
-    /**
-     * state wheel execute thread
-     */
     private final StateWheelExecuteThread stateWheelExecuteThread;
 
     private final String masterAddress;
@@ -267,14 +241,17 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      * the process start nodes are submitted completely.
      */
     public boolean isStart() {
-        return this.isStart;
+        return WorkflowRunnableStatus.STARTED == workflowRunnableStatus;
     }
 
     /**
      * handle event
      */
     public void handleEvents() {
-        if (!isStart) {
+        if (!isStart()) {
+            logger.info(
+                "The workflow instance is not started, will not handle its state event, current state event size: {}",
+                stateEvents);
             return;
         }
         StateEvent stateEvent = null;
@@ -378,45 +355,53 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     }
 
     public void taskFinished(TaskInstance taskInstance) throws StateEventHandleException {
-        logger.info("TaskInstance finished task code:{} state:{} ",
-                    taskInstance.getTaskCode(),
-                    taskInstance.getState());
-
-        activeTaskProcessorMaps.remove(taskInstance.getTaskCode());
-        stateWheelExecuteThread.removeTask4TimeoutCheck(processInstance, taskInstance);
-        stateWheelExecuteThread.removeTask4RetryCheck(processInstance, taskInstance);
-        stateWheelExecuteThread.removeTask4StateCheck(processInstance, taskInstance);
-
-        if (taskInstance.getState().typeIsSuccess()) {
-            completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
-            // todo: merge the last taskInstance
-            processInstance.setVarPool(taskInstance.getVarPool());
-            processService.saveProcessInstance(processInstance);
-            if (!processInstance.isBlocked()) {
-                submitPostNode(Long.toString(taskInstance.getTaskCode()));
-            }
-        } else if (taskInstance.taskCanRetry() && processInstance.getState() != ExecutionStatus.READY_STOP) {
-            // retry task
-            logger.info("Retry taskInstance taskInstance state: {}", taskInstance.getState());
-            retryTaskInstance(taskInstance);
-        } else if (taskInstance.getState().typeIsFailure()) {
-            completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
-            // There are child nodes and the failure policy is: CONTINUE
-            if (processInstance.getFailureStrategy() == FailureStrategy.CONTINUE
-                && DagHelper.haveAllNodeAfterNode(Long.toString(taskInstance.getTaskCode()), dag)) {
-                submitPostNode(Long.toString(taskInstance.getTaskCode()));
-            } else {
-                errorTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
-                if (processInstance.getFailureStrategy() == FailureStrategy.END) {
-                    killAllTasks();
+        logger.info("TaskInstance finished task code:{} state:{}", taskInstance.getTaskCode(), taskInstance.getState());
+        try {
+
+            activeTaskProcessorMaps.remove(taskInstance.getTaskCode());
+            stateWheelExecuteThread.removeTask4TimeoutCheck(processInstance, taskInstance);
+            stateWheelExecuteThread.removeTask4RetryCheck(processInstance, taskInstance);
+            stateWheelExecuteThread.removeTask4StateCheck(processInstance, taskInstance);
+
+            if (taskInstance.getState().typeIsSuccess()) {
+                completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
+                // todo: merge the last taskInstance
+                processInstance.setVarPool(taskInstance.getVarPool());
+                processService.saveProcessInstance(processInstance);
+                if (!processInstance.isBlocked()) {
+                    submitPostNode(Long.toString(taskInstance.getTaskCode()));
                 }
+            } else if (taskInstance.taskCanRetry() && processInstance.getState() != ExecutionStatus.READY_STOP) {
+                // retry task
+                logger.info("Retry taskInstance taskInstance state: {}", taskInstance.getState());
+                retryTaskInstance(taskInstance);
+            } else if (taskInstance.getState().typeIsFailure()) {
+                completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
+                // There are child nodes and the failure policy is: CONTINUE
+                if (processInstance.getFailureStrategy() == FailureStrategy.CONTINUE && DagHelper.haveAllNodeAfterNode(
+                    Long.toString(taskInstance.getTaskCode()),
+                    dag)) {
+                    submitPostNode(Long.toString(taskInstance.getTaskCode()));
+                } else {
+                    errorTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
+                    if (processInstance.getFailureStrategy() == FailureStrategy.END) {
+                        killAllTasks();
+                    }
+                }
+            } else if (taskInstance.getState().typeIsFinished()) {
+                // todo: when the task instance type is pause, then it should not in completeTaskMap
+                completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
             }
-        } else if (taskInstance.getState().typeIsFinished()) {
-            // todo: when the task instance type is pause, then it should not in completeTaskMap
-            completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
+            logger.info("TaskInstance finished will try to update the workflow instance state, task code:{} state:{}",
+                        taskInstance.getTaskCode(),
+                        taskInstance.getState());
+            this.updateProcessInstanceState();
+        } catch (Exception ex) {
+            logger.error("Task finish failed, get a exception, will remove this taskInstance from completeTaskMap", ex);
+            // remove the task from complete map, so that we can finish in the next time.
+            completeTaskMap.remove(taskInstance.getTaskCode());
+            throw ex;
         }
-
-        this.updateProcessInstanceState();
     }
 
     /**
@@ -674,17 +659,29 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      */
     @Override
     public WorkflowSubmitStatue call() {
-        if (this.taskInstanceMap.size() > 0 || isStart) {
-            logger.warn("The workflow has already been started");
+        if (isStart()) {
+            // This case should not been happened
+            logger.warn("[WorkflowInstance-{}] The workflow has already been started", processInstance.getId());
             return WorkflowSubmitStatue.DUPLICATED_SUBMITTED;
         }
 
         try {
             LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
-            buildFlowDag();
-            initTaskQueue();
-            submitPostNode(null);
-            isStart = true;
+            if (workflowRunnableStatus == WorkflowRunnableStatus.CREATED) {
+                buildFlowDag();
+                workflowRunnableStatus = WorkflowRunnableStatus.INITIALIZE_DAG;
+                logger.info("workflowStatue changed to :{}", workflowRunnableStatus);
+            }
+            if (workflowRunnableStatus == WorkflowRunnableStatus.INITIALIZE_DAG) {
+                initTaskQueue();
+                workflowRunnableStatus = WorkflowRunnableStatus.INITIALIZE_QUEUE;
+                logger.info("workflowStatue changed to :{}", workflowRunnableStatus);
+            }
+            if (workflowRunnableStatus == WorkflowRunnableStatus.INITIALIZE_QUEUE) {
+                submitPostNode(null);
+                workflowRunnableStatus = WorkflowRunnableStatus.STARTED;
+                logger.info("workflowStatue changed to :{}", workflowRunnableStatus);
+            }
             return WorkflowSubmitStatue.SUCCESS;
         } catch (Exception e) {
             logger.error("Start workflow error", e);
@@ -745,11 +742,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      * @throws Exception exception
      */
     private void buildFlowDag() throws Exception {
-        if (this.dag != null) {
-            return;
-        }
-        processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(),
-                                                                 processInstance.getProcessDefinitionVersion());
+        processDefinition = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(), processInstance.getProcessDefinitionVersion());
         processInstance.setProcessDefinition(processDefinition);
 
         List<TaskInstance> recoverNodeList = getRecoverTaskInstanceList(processInstance.getCommandParam());
@@ -880,9 +873,9 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             }
         }
         logger.info("Initialize task queue, dependFailedTaskMap: {}, completeTaskMap: {}, errorTaskMap: {}",
-            dependFailedTaskMap,
-            completeTaskMap,
-            errorTaskMap);
+                    dependFailedTaskMap,
+                    completeTaskMap,
+                    errorTaskMap);
     }
 
     /**
@@ -907,8 +900,10 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             boolean submit = taskProcessor.action(TaskAction.SUBMIT);
             if (!submit) {
                 logger.error("process id:{} name:{} submit standby task id:{} name:{} failed!",
-                        processInstance.getId(), processInstance.getName(),
-                        taskInstance.getId(), taskInstance.getName());
+                             processInstance.getId(),
+                             processInstance.getName(),
+                             taskInstance.getId(),
+                             taskInstance.getName());
                 return Optional.empty();
             }
 
@@ -959,7 +954,10 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             }
             return Optional.of(taskInstance);
         } catch (Exception e) {
-            logger.error("submit standby task error", e);
+            logger.error("submit standby task error, taskCode: {}, taskInstanceId: {}",
+                         taskInstance.getTaskCode(),
+                         taskInstance.getId(),
+                         e);
             return Optional.empty();
         }
     }
@@ -1208,9 +1206,19 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      */
     private Map<String, TaskInstance> getCompleteTaskInstanceMap() {
         Map<String, TaskInstance> completeTaskInstanceMap = new HashMap<>();
-        for (Integer taskInstanceId : completeTaskMap.values()) {
+        for (Map.Entry<Long, Integer> entry : completeTaskMap.entrySet()) {
+            Long taskConde = entry.getKey();
+            Integer taskInstanceId = entry.getValue();
             TaskInstance taskInstance = taskInstanceMap.get(taskInstanceId);
+            if (taskInstance == null) {
+                logger.warn("Cannot find the taskInstance from taskInstanceMap, taskInstanceId: {}, taskConde: {}",
+                            taskInstanceId,
+                            taskConde);
+                // This case will happen when we submit to db failed, then the taskInstanceId is 0
+                continue;
+            }
             completeTaskInstanceMap.put(Long.toString(taskInstance.getTaskCode()), taskInstance);
+
         }
         return completeTaskInstanceMap;
     }
@@ -1425,6 +1433,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      */
     private boolean processFailed() {
         if (hasFailedTask()) {
+            logger.info("The current process has failed task, the current process failed");
             if (processInstance.getFailureStrategy() == FailureStrategy.END) {
                 return true;
             }
@@ -1505,22 +1514,29 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
 
         if (activeTaskProcessorMaps.size() > 0 || hasRetryTaskInStandBy()) {
             // active task and retry task exists
-            return runningState(state);
+            ExecutionStatus executionStatus = runningState(state);
+            logger.info("The workflowInstance has task running, the workflowInstance status is {}", executionStatus);
+            return executionStatus;
         }
 
         // block
         if (state == ExecutionStatus.READY_BLOCK) {
-            return processReadyBlock();
+            ExecutionStatus executionStatus = processReadyBlock();
+            logger.info("The workflowInstance is ready to block, the workflowInstance status is {}", executionStatus);
         }
 
         // waiting thread
         if (hasWaitingThreadTask()) {
+            logger.info("The workflowInstance has waiting thread task, the workflow status is {}",
+                        ExecutionStatus.WAITING_THREAD);
             return ExecutionStatus.WAITING_THREAD;
         }
 
         // pause
         if (state == ExecutionStatus.READY_PAUSE) {
-            return processReadyPause();
+            ExecutionStatus executionStatus = processReadyPause();
+            logger.info("The workflowInstance is ready to pause, the workflow status is {}", executionStatus);
+            return executionStatus;
         }
 
         // stop
@@ -1528,18 +1544,18 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             List<TaskInstance> stopList = getCompleteTaskByState(ExecutionStatus.STOP);
             List<TaskInstance> killList = getCompleteTaskByState(ExecutionStatus.KILL);
             List<TaskInstance> failList = getCompleteTaskByState(ExecutionStatus.FAILURE);
-            if (CollectionUtils.isNotEmpty(stopList)
-                || CollectionUtils.isNotEmpty(killList)
-                || CollectionUtils.isNotEmpty(failList)
-                || !isComplementEnd()) {
-                return ExecutionStatus.STOP;
+            ExecutionStatus executionStatus;
+            if (CollectionUtils.isNotEmpty(stopList) || CollectionUtils.isNotEmpty(killList) || CollectionUtils.isNotEmpty(failList) || !isComplementEnd()) {
+                executionStatus = ExecutionStatus.STOP;
             } else {
-                return ExecutionStatus.SUCCESS;
+                executionStatus = ExecutionStatus.SUCCESS;
             }
+            logger.info("The workflowInstance is ready to stop, the workflow status is {}", executionStatus);
         }
 
         // process failure
         if (processFailed()) {
+            logger.info("The workflowInstance is failed, the workflow status is {}", ExecutionStatus.FAILURE);
             return ExecutionStatus.FAILURE;
         }
 
@@ -1583,15 +1599,21 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     private void updateProcessInstanceState() throws StateEventHandleException {
         ExecutionStatus state = getProcessInstanceState(processInstance);
         if (processInstance.getState() != state) {
+            logger.info("Update workflowInstance states, origin state: {}, target state: {}",
+                        processInstance.getState(),
+                        state);
             updateWorkflowInstanceStatesToDB(state);
 
             StateEvent stateEvent = new StateEvent();
             stateEvent.setExecutionStatus(processInstance.getState());
             stateEvent.setProcessInstanceId(this.processInstance.getId());
             stateEvent.setType(StateEventType.PROCESS_STATE_CHANGE);
-            // this.processStateChangeHandler(stateEvent);
             // replace with `stateEvents`, make sure `WorkflowExecuteThread` can be deleted to avoid memory leaks
             this.stateEvents.add(stateEvent);
+        } else {
+            logger.info("There is no need to update the workflow instance state, origin state: {}, target state: {}",
+                        processInstance.getState(),
+                        state);
         }
     }
 
@@ -1606,12 +1628,9 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     private void updateWorkflowInstanceStatesToDB(ExecutionStatus newStates) throws StateEventHandleException {
         ExecutionStatus originStates = processInstance.getState();
         if (originStates != newStates) {
-            logger.info("work flow process instance [id: {}, name:{}], state change from {} to {}, cmd type: {}",
-                processInstance.getId(),
-                processInstance.getName(),
-                originStates,
-                newStates,
-                processInstance.getCommandType());
+            logger.info("Begin to update workflow instance state , state will change from {} to {}",
+                        originStates,
+                        newStates);
 
             processInstance.setState(newStates);
             if (newStates.typeIsFinished()) {
@@ -1661,8 +1680,8 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
      *
      * @param taskInstance task instance
      */
-    private void removeTaskFromStandbyList(TaskInstance taskInstance) {
-        readyToSubmitTaskQueue.remove(taskInstance);
+    private boolean removeTaskFromStandbyList(TaskInstance taskInstance) {
+        return readyToSubmitTaskQueue.remove(taskInstance);
     }
 
     /**
@@ -1753,12 +1772,20 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
                 if (!taskInstanceOptional.isPresent()) {
                     this.taskFailedSubmit = true;
                     // Remove and add to complete map and error map
-                    removeTaskFromStandbyList(task);
+                    if (!removeTaskFromStandbyList(task)) {
+                        logger.error(
+                            "Task submit failed, remove from standby list failed, workflowInstanceId: {}, taskCode: {}",
+                            processInstance.getId(),
+                            task.getTaskCode());
+                    }
                     completeTaskMap.put(task.getTaskCode(), task.getId());
+                    taskInstanceMap.put(task.getId(), task);
                     errorTaskMap.put(task.getTaskCode(), task.getId());
-                    logger.error("Task submitted failed, processInstanceId: {}, taskInstanceId: {}",
+                    activeTaskProcessorMaps.remove(task.getTaskCode());
+                    logger.error("Task submitted failed, workflowInstanceId: {}, taskInstanceId: {}, taskCode: {}",
                                  task.getProcessInstanceId(),
-                                 task.getId());
+                                 task.getId(),
+                                 task.getTaskCode());
                 } else {
                     removeTaskFromStandbyList(task);
                 }
@@ -1930,4 +1957,10 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         }
     }
 
+    private enum WorkflowRunnableStatus {
+        CREATED, INITIALIZE_DAG, INITIALIZE_QUEUE, STARTED,
+        ;
+
+    }
+
 }
\ No newline at end of file
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRecallCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskRecallCommand.java
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 5624f47812..b12cd77de2 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -1250,8 +1250,8 @@ public class ProcessServiceImpl implements ProcessService {
                     break;
                 }
                 logger.error(
-                    "task commit to db failed , taskId {} has already retry {} times, please check the database",
-                    taskInstance.getId(),
+                    "task commit to db failed , taskCode: {} has already retry {} times, please check the database",
+                    taskInstance.getTaskCode(),
                     retryTimes);
                 Thread.sleep(commitInterval);
             } catch (Exception e) {
@@ -1264,6 +1264,7 @@ public class ProcessServiceImpl implements ProcessService {
     }
 
     /**
+     * // todo: This method need to refactor, we find when the db down, but the taskInstanceId is not 0. It's better to change to void, rather than return TaskInstance
      * submit task to db
      * submit sub process to command
      *
@@ -1283,7 +1284,7 @@ public class ProcessServiceImpl implements ProcessService {
         if (task == null) {
             logger.error("Save taskInstance to db error, task name:{}, process id:{} state: {} ",
                          taskInstance.getName(),
-                         taskInstance.getProcessInstance(),
+                         taskInstance.getProcessInstance().getId(),
                          processInstance.getState());
             return null;
         }
@@ -1294,8 +1295,8 @@ public class ProcessServiceImpl implements ProcessService {
 
         logger.info(
             "End save taskInstance to db successfully:{}, taskInstanceName: {}, taskInstance state:{}, processInstanceId:{}, processInstanceState: {}",
-            taskInstance.getId(),
-            taskInstance.getName(),
+            task.getId(),
+            task.getName(),
             task.getState(),
             processInstance.getId(),
             processInstance.getState());
@@ -1524,7 +1525,10 @@ public class ProcessServiceImpl implements ProcessService {
     public TaskInstance submitTaskInstanceToDB(TaskInstance taskInstance, ProcessInstance processInstance) {
         ExecutionStatus processInstanceState = processInstance.getState();
         if (processInstanceState.typeIsFinished() || processInstanceState == ExecutionStatus.READY_STOP) {
-            logger.warn("processInstance {} was {}, skip submit task", processInstance.getProcessDefinitionCode(), processInstanceState);
+            logger.warn("processInstance: {} state was: {}, skip submit this task, taskCode: {}",
+                        processInstance.getId(),
+                        processInstanceState,
+                        taskInstance.getTaskCode());
             return null;
         }
         if (processInstanceState == ExecutionStatus.READY_PAUSE) {
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java
index f28990b152..a261fec0a1 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/cache/ResponseCache.java
@@ -17,7 +17,7 @@
 
 package org.apache.dolphinscheduler.server.worker.cache;
 
-import org.apache.dolphinscheduler.common.enums.Event;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.remote.command.Command;
 
 import java.util.Map;
@@ -47,7 +47,7 @@ public class ResponseCache {
      * @param command command
      * @param event event ACK/RESULT
      */
-    public void cache(Integer taskInstanceId, Command command, Event event) {
+    public void cache(Integer taskInstanceId, Command command, TaskEventType event) {
         switch (event) {
             case RUNNING:
                 runningCache.put(taskInstanceId, command);
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
index 5a9e2bcef6..626cc6f7e5 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
@@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.server.worker.processor;
 
 import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS;
 
-import org.apache.dolphinscheduler.common.enums.Event;
+import org.apache.dolphinscheduler.common.enums.TaskEventType;
 import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.remote.NettyRemotingClient;
@@ -227,7 +227,7 @@ public class TaskCallbackService {
     public void sendTaskExecuteRunningCommand(TaskExecutionContext taskExecutionContext) {
         TaskExecuteRunningCommand command = buildTaskExecuteRunningCommand(taskExecutionContext);
         // add response cache
-        ResponseCache.get().cache(taskExecutionContext.getTaskInstanceId(), command.convert2Command(), Event.RUNNING);
+        ResponseCache.get().cache(taskExecutionContext.getTaskInstanceId(), command.convert2Command(), TaskEventType.RUNNING);
         send(taskExecutionContext.getTaskInstanceId(), command.convert2Command());
     }
 
@@ -247,7 +247,7 @@ public class TaskCallbackService {
     public void sendTaskExecuteResponseCommand(TaskExecutionContext taskExecutionContext) {
         TaskExecuteResponseCommand command = buildTaskExecuteResponseCommand(taskExecutionContext);
         // add response cache
-        ResponseCache.get().cache(taskExecutionContext.getTaskInstanceId(), command.convert2Command(), Event.RESULT);
+        ResponseCache.get().cache(taskExecutionContext.getTaskInstanceId(), command.convert2Command(), TaskEventType.RESULT);
         send(taskExecutionContext.getTaskInstanceId(), command.convert2Command());
     }
 
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
index 108ae8a3bb..caa25ebbf5 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
@@ -83,34 +83,35 @@ public class RetryReportTaskStatusThread extends BaseDaemonThread {
     private void retryRunningCommand(ResponseCache instance) {
         if (!instance.getRunningCache().isEmpty()) {
             Map<Integer, Command> runningCache = instance.getRunningCache();
+            logger.info("Send task running retry command starting, waiting to retry size: {}", runningCache.size());
             for (Map.Entry<Integer, Command> entry : runningCache.entrySet()) {
                 Integer taskInstanceId = entry.getKey();
                 Command runningCommand = entry.getValue();
                 try {
                     taskCallbackService.send(taskInstanceId, runningCommand);
                 } catch (Exception ex) {
-                    logger.error("Retry send running command to master error, taskInstanceId: {}, command: {}",
-                                 taskInstanceId,
-                                 runningCommand);
+                    logger.error("Retry send running command to master error, taskInstanceId: {}, command: {}", taskInstanceId, runningCommand);
                 }
             }
+            logger.info("Send task running retry command finished, waiting to retry size: {}", runningCache.size());
         }
     }
 
     private void retryResponseCommand(ResponseCache instance) {
-        if (!instance.getResponseCache().isEmpty()) {
-            Map<Integer, Command> responseCache = instance.getResponseCache();
+        Map<Integer, Command> responseCache = instance.getResponseCache();
+        if (!responseCache.isEmpty()) {
+            logger.info("Send task response retry command starting, waiting to retry size: {}", responseCache.size());
             for (Map.Entry<Integer, Command> entry : responseCache.entrySet()) {
                 Integer taskInstanceId = entry.getKey();
                 Command responseCommand = entry.getValue();
                 try {
                     taskCallbackService.send(taskInstanceId, responseCommand);
                 } catch (Exception ex) {
-                    logger.error("Retry send response command to master error, taskInstanceId: {}, command: {}",
-                                 taskInstanceId,
-                                 responseCommand);
+                    logger.error("Retry send response command to master error, taskInstanceId: {}, command: {}", taskInstanceId, responseCommand);
                 }
             }
+            logger.info("Send task response retry command finished, waiting to retry size: {}", responseCache.size());
         }
     }
+
 }


[dolphinscheduler] 26/29: Fix kill yarn job error when failover caused by doesn't set ProcessDefinition (#10948)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 44ddb6908edba75163485fcaed7156d75bb17b3f
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Wed Jul 13 22:32:30 2022 +0800

    Fix kill yarn job error when failover caused by doesn't set ProcessDefinition (#10948)
    
    (cherry picked from commit b245e7c973a5282a1405097605a4afacac1b5eab)
---
 .../master/service/MasterFailoverService.java       | 21 +++++++++++++++------
 .../master/service/WorkerFailoverService.java       |  9 +++++----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
index c7e5b4ea13..61ba7c3fd6 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/MasterFailoverService.java
@@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.enums.NodeType;
 import org.apache.dolphinscheduler.common.model.Server;
 import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -114,16 +115,18 @@ public class MasterFailoverService {
      * @param masterHost master host
      */
     private void doFailoverMaster(@NonNull String masterHost) {
-        LOGGER.info("Master[{}] failover starting, need to failover process", masterHost);
         StopWatch failoverTimeCost = StopWatch.createStarted();
 
-        Optional<Date> masterStartupTimeOptional =
-            getServerStartupTime(registryClient.getServerList(NodeType.MASTER), masterHost);
-        List<ProcessInstance> needFailoverProcessInstanceList =
-            processService.queryNeedFailoverProcessInstances(masterHost);
+        Optional<Date> masterStartupTimeOptional = getServerStartupTime(registryClient.getServerList(NodeType.MASTER),
+                                                                        masterHost);
+        List<ProcessInstance> needFailoverProcessInstanceList = processService.queryNeedFailoverProcessInstances(
+            masterHost);
+        if (CollectionUtils.isEmpty(needFailoverProcessInstanceList)) {
+            return;
+        }
 
         LOGGER.info(
-            "Master[{}] failover there are {} workflowInstance may need to failover, will do a deep check, workflowInstanceIds: {}",
+            "Master[{}] failover starting there are {} workflowInstance may need to failover, will do a deep check, workflowInstanceIds: {}",
             masterHost,
             needFailoverProcessInstanceList.size(),
             needFailoverProcessInstanceList.stream().map(ProcessInstance::getId).collect(Collectors.toList()));
@@ -136,6 +139,11 @@ public class MasterFailoverService {
                     LOGGER.info("WorkflowInstance doesn't need to failover");
                     continue;
                 }
+                // todo: use batch query
+                ProcessDefinition processDefinition
+                    = processService.findProcessDefinition(processInstance.getProcessDefinitionCode(),
+                                                           processInstance.getProcessDefinitionVersion());
+                processInstance.setProcessDefinition(processDefinition);
                 int processInstanceId = processInstance.getId();
                 List<TaskInstance> taskInstanceList = processService.findValidTaskListByProcessId(processInstanceId);
                 for (TaskInstance taskInstance : taskInstanceList) {
@@ -205,6 +213,7 @@ public class MasterFailoverService {
             TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
                 .buildTaskInstanceRelatedInfo(taskInstance)
                 .buildProcessInstanceRelatedInfo(processInstance)
+                .buildProcessDefinitionRelatedInfo(processInstance.getProcessDefinition())
                 .create();
 
             if (masterConfig.isKillYarnJobWhenTaskFailover()) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java
index ec126a3ec3..402ec43354 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/WorkerFailoverService.java
@@ -115,10 +115,10 @@ public class WorkerFailoverService {
         for (TaskInstance taskInstance : needFailoverTaskInstanceList) {
             LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskInstance.getProcessInstanceId(), taskInstance.getId());
             try {
-                ProcessInstance processInstance =
-                    processInstanceCacheMap.computeIfAbsent(taskInstance.getProcessInstanceId(), k -> {
-                        WorkflowExecuteRunnable workflowExecuteRunnable =
-                            cacheManager.getByProcessInstanceId(taskInstance.getProcessInstanceId());
+                ProcessInstance processInstance = processInstanceCacheMap.computeIfAbsent(
+                    taskInstance.getProcessInstanceId(), k -> {
+                        WorkflowExecuteRunnable workflowExecuteRunnable = cacheManager.getByProcessInstanceId(
+                            taskInstance.getProcessInstanceId());
                         if (workflowExecuteRunnable == null) {
                             return null;
                         }
@@ -167,6 +167,7 @@ public class WorkerFailoverService {
             TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
                 .buildTaskInstanceRelatedInfo(taskInstance)
                 .buildProcessInstanceRelatedInfo(processInstance)
+                .buildProcessDefinitionRelatedInfo(processInstance.getProcessDefinition())
                 .create();
 
             if (masterConfig.isKillYarnJobWhenTaskFailover()) {


[dolphinscheduler] 09/29: Remove the schedule thread in LowerWeightHostManager (#10310)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 9a28d320570e9c6768382584a97ab9d2d9225d96
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jun 21 13:30:07 2022 +0800

    Remove the schedule thread in LowerWeightHostManager (#10310)
    
    (cherry picked from commit b100f6c4890fb8db8887e24891c6af963c582360)
---
 .../dispatch/host/LowerWeightHostManager.java      | 136 ++++++++++-----------
 .../server/master/registry/ServerNodeManager.java  |  22 ++++
 .../master/registry/WorkerInfoChangeListener.java} |  37 ++----
 3 files changed, 100 insertions(+), 95 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
index 646d770b01..f7c26ae765 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
@@ -20,11 +20,11 @@ package org.apache.dolphinscheduler.server.master.dispatch.host;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.utils.HeartBeat;
 import org.apache.dolphinscheduler.remote.utils.Host;
-import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
 import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
 import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWeight;
 import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker;
 import org.apache.dolphinscheduler.server.master.dispatch.host.assign.LowerWeightRoundRobin;
+import org.apache.dolphinscheduler.server.master.registry.WorkerInfoChangeListener;
 import org.apache.dolphinscheduler.spi.utils.StringUtils;
 
 import org.apache.commons.collections.CollectionUtils;
@@ -36,14 +36,10 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 import javax.annotation.PostConstruct;
-import javax.annotation.PreDestroy;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -70,23 +66,12 @@ public class LowerWeightHostManager extends CommonHostManager {
      */
     private Lock lock;
 
-    /**
-     * executor service
-     */
-    private ScheduledExecutorService executorService;
-
     @PostConstruct
     public void init() {
         this.selector = new LowerWeightRoundRobin();
         this.workerHostWeightsMap = new ConcurrentHashMap<>();
         this.lock = new ReentrantLock();
-        this.executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("LowerWeightHostManagerExecutor"));
-        this.executorService.scheduleWithFixedDelay(new RefreshResourceTask(), 0, 1, TimeUnit.SECONDS);
-    }
-
-    @PreDestroy
-    public void close() {
-        this.executorService.shutdownNow();
+        serverNodeManager.addWorkerInfoChangeListener(new WorkerWeightListener());
     }
 
     /**
@@ -109,6 +94,69 @@ public class LowerWeightHostManager extends CommonHostManager {
         throw new UnsupportedOperationException("not support");
     }
 
+
+    private class WorkerWeightListener implements WorkerInfoChangeListener {
+        @Override
+        public void notify(Map<String, Set<String>> workerGroups, Map<String, String> workerNodeInfo) {
+            syncWorkerResources(workerGroups, workerNodeInfo);
+        }
+    }
+
+    /**
+     * Sync worker resource.
+     *
+     * @param workerGroupNodes  worker group nodes, key is worker group, value is worker group nodes.
+     * @param workerNodeInfoMap worker node info map, key is worker node, value is worker info.
+     */
+    private void syncWorkerResources(final Map<String, Set<String>> workerGroupNodes,
+                                     final Map<String, String> workerNodeInfoMap) {
+        try {
+            Map<String, Set<HostWeight>> workerHostWeights = new HashMap<>();
+            for (Map.Entry<String, Set<String>> entry : workerGroupNodes.entrySet()) {
+                String workerGroup = entry.getKey();
+                Set<String> nodes = entry.getValue();
+                Set<HostWeight> hostWeights = new HashSet<>(nodes.size());
+                for (String node : nodes) {
+                    String heartbeat = workerNodeInfoMap.getOrDefault(node, null);
+                    Optional<HostWeight> hostWeightOpt = getHostWeight(node, workerGroup, heartbeat);
+                    hostWeightOpt.ifPresent(hostWeights::add);
+                }
+                if (!hostWeights.isEmpty()) {
+                    workerHostWeights.put(workerGroup, hostWeights);
+                }
+            }
+            syncWorkerHostWeight(workerHostWeights);
+        } catch (Throwable ex) {
+            logger.error("Sync worker resource error", ex);
+        }
+    }
+
+    private Optional<HostWeight> getHostWeight(String addr, String workerGroup, String heartBeatInfo) {
+        if (StringUtils.isEmpty(heartBeatInfo)) {
+            logger.warn("worker {} in work group {} have not received the heartbeat", addr, workerGroup);
+            return Optional.empty();
+        }
+        HeartBeat heartBeat = HeartBeat.decodeHeartBeat(heartBeatInfo);
+        if (heartBeat == null) {
+            return Optional.empty();
+        }
+        if (Constants.ABNORMAL_NODE_STATUS == heartBeat.getServerStatus()) {
+            logger.warn("worker {} current cpu load average {} is too high or available memory {}G is too low",
+                    addr, heartBeat.getLoadAverage(), heartBeat.getAvailablePhysicalMemorySize());
+            return Optional.empty();
+        }
+        if (Constants.BUSY_NODE_STATUE == heartBeat.getServerStatus()) {
+            logger.warn("worker {} is busy, current waiting task count {} is large than worker thread count {}",
+                    addr, heartBeat.getWorkerWaitingTaskCount(), heartBeat.getWorkerExecThreadCount());
+            return Optional.empty();
+        }
+        return Optional.of(
+            new HostWeight(HostWorker.of(addr, heartBeat.getWorkerHostWeight(), workerGroup),
+                           heartBeat.getCpuUsage(), heartBeat.getMemoryUsage(), heartBeat.getLoadAverage(),
+                           heartBeat.getStartupTime()));
+    }
+
+
     private void syncWorkerHostWeight(Map<String, Set<HostWeight>> workerHostWeights) {
         lock.lock();
         try {
@@ -128,58 +176,4 @@ public class LowerWeightHostManager extends CommonHostManager {
         }
     }
 
-    class RefreshResourceTask implements Runnable {
-
-        @Override
-        public void run() {
-            try {
-                Map<String, Set<HostWeight>> workerHostWeights = new HashMap<>();
-                Map<String, Set<String>> workerGroupNodes = serverNodeManager.getWorkerGroupNodes();
-                for (Map.Entry<String, Set<String>> entry : workerGroupNodes.entrySet()) {
-                    String workerGroup = entry.getKey();
-                    Set<String> nodes = entry.getValue();
-                    Set<HostWeight> hostWeights = new HashSet<>(nodes.size());
-                    for (String node : nodes) {
-                        String heartbeat = serverNodeManager.getWorkerNodeInfo(node);
-                        Optional<HostWeight> hostWeightOpt = getHostWeight(node, workerGroup, heartbeat);
-                        if (hostWeightOpt.isPresent()) {
-                            hostWeights.add(hostWeightOpt.get());
-                        }
-                    }
-                    if (!hostWeights.isEmpty()) {
-                        workerHostWeights.put(workerGroup, hostWeights);
-                    }
-                }
-                syncWorkerHostWeight(workerHostWeights);
-            } catch (Throwable ex) {
-                logger.error("RefreshResourceTask error", ex);
-            }
-        }
-
-        public Optional<HostWeight> getHostWeight(String addr, String workerGroup, String heartBeatInfo) {
-            if (StringUtils.isEmpty(heartBeatInfo)) {
-                logger.warn("worker {} in work group {} have not received the heartbeat", addr, workerGroup);
-                return Optional.empty();
-            }
-            HeartBeat heartBeat = HeartBeat.decodeHeartBeat(heartBeatInfo);
-            if (heartBeat == null) {
-                return Optional.empty();
-            }
-            if (Constants.ABNORMAL_NODE_STATUS == heartBeat.getServerStatus()) {
-                logger.warn("worker {} current cpu load average {} is too high or available memory {}G is too low",
-                        addr, heartBeat.getLoadAverage(), heartBeat.getAvailablePhysicalMemorySize());
-                return Optional.empty();
-            }
-            if (Constants.BUSY_NODE_STATUE == heartBeat.getServerStatus()) {
-                logger.warn("worker {} is busy, current waiting task count {} is large than worker thread count {}",
-                        addr, heartBeat.getWorkerWaitingTaskCount(), heartBeat.getWorkerExecThreadCount());
-                return Optional.empty();
-            }
-            return Optional.of(
-                    new HostWeight(HostWorker.of(addr, heartBeat.getWorkerHostWeight(), workerGroup),
-                            heartBeat.getCpuUsage(), heartBeat.getMemoryUsage(), heartBeat.getLoadAverage(),
-                            heartBeat.getStartupTime()));
-        }
-    }
-
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
index a6599b6032..0c550a1ee3 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
@@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.service.registry.RegistryClient;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -131,6 +132,8 @@ public class ServerNodeManager implements InitializingBean {
     @Autowired
     private MasterConfig masterConfig;
 
+    private List<WorkerInfoChangeListener> workerInfoChangeListeners = new ArrayList<>();
+
     private static volatile int MASTER_SLOT = 0;
 
     private static volatile int MASTER_SIZE = 0;
@@ -217,6 +220,7 @@ public class ServerNodeManager implements InitializingBean {
                         }
                     }
                 }
+                notifyWorkerInfoChangeListeners();
             } catch (Exception e) {
                 logger.error("WorkerNodeInfoAndGroupDbSyncTask error:", e);
             }
@@ -256,6 +260,7 @@ public class ServerNodeManager implements InitializingBean {
                         String node = parseNode(path);
                         syncSingleWorkerNodeInfo(node, data);
                     }
+                    notifyWorkerInfoChangeListeners();
                 } catch (IllegalArgumentException ex) {
                     logger.warn(ex.getMessage());
                 } catch (Exception ex) {
@@ -457,6 +462,23 @@ public class ServerNodeManager implements InitializingBean {
         }
     }
 
+    /**
+     * Add the resource change listener, when the resource changed, the listener will be notified.
+     *
+     * @param listener will be trigger, when the worker node info changed.
+     */
+    public synchronized void addWorkerInfoChangeListener(WorkerInfoChangeListener listener) {
+        workerInfoChangeListeners.add(listener);
+    }
+
+    private void notifyWorkerInfoChangeListeners() {
+        Map<String, Set<String>> workerGroupNodes = getWorkerGroupNodes();
+        Map<String, String> workerNodeInfo = getWorkerNodeInfo();
+        for (WorkerInfoChangeListener listener : workerInfoChangeListeners) {
+            listener.notify(workerGroupNodes, workerNodeInfo);
+        }
+    }
+
     /**
      * destroy
      */
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/WorkerInfoChangeListener.java
similarity index 50%
rename from dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/WorkerInfoChangeListener.java
index 5e35b0b2ec..f885a6fba0 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RefreshResourceTaskTest.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/WorkerInfoChangeListener.java
@@ -15,33 +15,22 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.server.master.dispatch.host;
+package org.apache.dolphinscheduler.server.master.registry;
 
-import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.junit.MockitoJUnitRunner;
+import java.util.Map;
+import java.util.Set;
 
 /**
- * RefreshResourceTask test
+ * The listener used in {@link ServerNodeManager} to notify the change of worker info.
  */
-@RunWith(MockitoJUnitRunner.class)
-public class RefreshResourceTaskTest {
-
-    @Mock
-    private ServerNodeManager serverNodeManager;
+public interface WorkerInfoChangeListener {
 
-    @InjectMocks
-    LowerWeightHostManager lowerWeightHostManager;
+    /**
+     * Used to notify the change of worker info.
+     *
+     * @param workerGroups   worker groups map, key is worker group name, value is worker address.
+     * @param workerNodeInfo worker node info map, key is worker address, value is worker info.
+     */
+    void notify(Map<String, Set<String>> workerGroups, Map<String, String> workerNodeInfo);
 
-    @Test
-    public void testGetHostWeightWithResult() {
-        Assert.assertTrue(!lowerWeightHostManager.new RefreshResourceTask()
-            .getHostWeight("192.168.1.1:22", "default", null)
-            .isPresent());
-    }
-}
\ No newline at end of file
+}


[dolphinscheduler] 03/29: [Feature][metrics] Add master, worker metrics (#10326)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 318a8e3ae0b5513ef8ebb60ad66dd701946497e1
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Thu Jun 9 10:55:39 2022 +0800

    [Feature][metrics] Add master, worker metrics (#10326)
    
    * Add mater metrics
    
    * fix UT
    
    * Add url to mysql profile
    
    * Add worker metrics
    
    * Update grafana config
    
    * Add system metrics doc
    
    * Add process failover counter
    
    * Add metrics image
    
    * Change jpg to png
    
    * Add command insert metrics
    
    * Fix UT
    
    * Revert UT
    
    (cherry picked from commit e21d7b1551c891caa5aeb8d3d61407e8cced7107)
---
 docs/docs/en/guide/metrics/metrics.md              |  154 ++
 docs/img/metrics/metrics-datasource.png            |  Bin 0 -> 336581 bytes
 docs/img/metrics/metrics-master.png                |  Bin 0 -> 467778 bytes
 docs/img/metrics/metrics-worker.png                |  Bin 0 -> 403432 bytes
 .../src/main/resources/banner.txt                  |    2 +-
 .../src/main/resources/application.yaml            |    3 +
 dolphinscheduler-api/src/main/resources/banner.txt |    2 +-
 .../apache/dolphinscheduler/common/Constants.java  |    8 +-
 .../dolphinscheduler/common/model/TaskNode.java    |    2 +-
 .../impl/ProcessInstanceExecCacheManagerImpl.java  |    8 +
 .../master/consumer/TaskPriorityQueueConsumer.java |   13 +-
 .../server/master/metrics/MasterServerMetrics.java |   53 +
 .../master/metrics/ProcessInstanceMetrics.java     |  101 +
 .../server/master/metrics/TaskMetrics.java         |  137 +
 .../master/runner/FailoverExecuteThread.java       |    2 +-
 .../master/runner/MasterSchedulerService.java      |    4 +-
 .../master/runner/WorkflowExecuteRunnable.java     |   82 +-
 .../master/runner/WorkflowExecuteThreadPool.java   |   16 +-
 .../master/runner/task/CommonTaskProcessor.java    |    6 +-
 .../master/runner/task/TaskProcessorFactory.java   |    7 +-
 .../server/master/service/FailoverService.java     |    5 +-
 .../src/main/resources/application.yaml            |    3 +
 .../src/main/resources/banner.txt                  |    2 +-
 .../server/master/WorkflowExecuteTaskTest.java     |    4 -
 .../master/registry/ServerNodeManagerTest.java     |   60 -
 .../dolphinscheduler/meter/MeterConfiguration.java |   16 +
 .../resources/grafana-demo/docker-compose.yaml     |    5 +-
 .../resources/grafana/DolphinSchedulerMaster.json  | 2607 ++++++++++++++++----
 .../resources/grafana/DolphinSchedulerWorker.json  | 1598 ++++++++++++
 .../processor/StateEventCallbackService.java       |    2 +-
 .../service/process/ProcessServiceImpl.java        |    2 +
 .../src/main/resources/banner.txt                  |    2 +-
 .../server/worker/metrics/TaskMetrics.java         |   58 +
 .../server/worker/metrics/WorkerServerMetrics.java |   56 +
 .../worker/processor/TaskCallbackService.java      |    2 +-
 .../worker/processor/TaskExecuteProcessor.java     |    8 +
 .../server/worker/runner/WorkerExecService.java    |    3 +
 .../server/worker/runner/WorkerManagerThread.java  |    1 +
 .../src/main/resources/application.yaml            |    3 +
 .../src/main/resources/banner.txt                  |    2 +-
 40 files changed, 4466 insertions(+), 573 deletions(-)

diff --git a/docs/docs/en/guide/metrics/metrics.md b/docs/docs/en/guide/metrics/metrics.md
new file mode 100644
index 0000000000..0f75db0ef4
--- /dev/null
+++ b/docs/docs/en/guide/metrics/metrics.md
@@ -0,0 +1,154 @@
+# Introduction
+
+Apache DolphinScheduler has export some metrics to monitor the system. We use micrometer for the exporter facade, and
+the default exporter is prometheus, more exporter is coming soon.
+
+## Quick Start
+
+You can add the following config in master/worker/alert/api's yaml file to open the metrics exporter.
+
+```yaml
+metrics:
+  enabled: true
+```
+
+Once you open the metrics exporter, you can access the metrics by the url: `http://ip:port/actuator/prometheus`
+
+The exporter port is the `server.port` defined in application.yaml, e.g: master: `server.port: 5679`, worker: `server.port: 1235`, alert: `server.port: 50053`, api: `server.port: 12345`.
+
+For example, you can get the master metrics by `curl http://localhost:5679/actuator/prometheus`
+
+We have prepared the out-of-the-box Grafana configuration for you, you can find the Grafana dashboard
+at `dolphinscheduler-meter/resources/grafana`, you can directly import these dashboards to grafana.
+
+If you want to try at docker, you can use the following command to start the prometheus with grafana:
+
+```shell
+cd dolphinscheduler-meter/src/main/resources/grafana-demo
+docker compose up
+```
+
+Then you can access the grafana by the url: `http://localhost/3001`
+
+![image.png](../../../../img/metrics/metrics-master.png)
+![image.png](../../../../img/metrics/metrics-worker.png)
+![image.png](../../../../img/metrics/metrics-datasource.png)
+
+## Master Metrics
+
+Master metrics are exported by the DolphinScheduler master server.
+
+### System Metrics
+
+* dolphinscheduler_master_overload_count: Indicates the number of times the master has been overloaded.
+* dolphinscheduler_master_consume_command_count: Indicates the number of commands has consumed.
+
+### Process Metrics
+
+* dolphinscheduler_create_command_count: Indicates the number of command has been inserted.
+* dolphinscheduler_process_instance_submit_count: Indicates the number of process has been submitted.
+* dolphinscheduler_process_instance_running_gauge: Indicates the number of process are running now.
+* dolphinscheduler_process_instance_timeout_count: Indicates the number of process has been timeout.
+* dolphinscheduler_process_instance_finish_count: Indicates the number of process has been finished, include success or
+  failure.
+* dolphinscheduler_process_instance_success_count: Indicates the number of process has been successful.
+* dolphinscheduler_process_instance_stop_count: Indicates the number of process has been stopped.
+* dolphinscheduler_process_instance_failover_count: Indicates the number of process has been failed over.
+
+### Task Metrics
+
+* dolphinscheduler_task_timeout_count: Indicates the number of tasks has been timeout.
+* dolphinscheduler_task_finish_count: Indicates the number of tasks has been finished, include success or failure.
+* dolphinscheduler_task_success_count: Indicates the number of tasks has been successful.
+* dolphinscheduler_task_timeout_count: Indicates the number of tasks has been timeout.
+* dolphinscheduler_task_retry_count: Indicates the number of tasks has been retry.
+* dolphinscheduler_task_failover_count: Indicates the number of tasks has been failover.
+* dolphinscheduler_task_dispatch_count: Indicates the number of tasks has been dispatched to worker.
+* dolphinscheduler_task_dispatch_failed_count: Indicates the number of tasks dispatched failed, if dispatched failed
+  will retry.
+* dolphinscheduler_task_dispatch_error_count: Indicates the number of tasks dispatched error, if dispatched error, means
+  there are exception occur.
+
+## Worker Metrics
+
+Worker metrics are exported by the DolphinScheduler worker server.
+
+### System Metrics
+
+* dolphinscheduler_worker_overload_count: Indicates the number of times the worker has been overloaded.
+* dolphinscheduler_worker_submit_queue_is_full_count: Indicates the number of times the worker's submit queue has been
+  full.
+
+### Task Metrics
+
+* dolphinscheduler_task_execute_count: Indicates the number of times a task has been executed, it contains a tag -
+  `task_type`.
+* dolphinscheduler_task_execution_count: Indicates the total number of task has been executed.
+* dolphinscheduler_task_execution_timer: Indicates the time spent executing tasks.
+
+## Default System Metrics
+
+In each server, there are some default metrics related to the system instance.
+
+### Database Metrics
+
+* hikaricp_connections_creation_seconds_max: Connection creation time max.
+* hikaricp_connections_creation_seconds_count: Connection creation time count.
+* hikaricp_connections_creation_seconds_sum: Connection creation time sum.
+* hikaricp_connections_acquire_seconds_max: Connection acquire time max.
+* hikaricp_connections_acquire_seconds_count: Connection acquire time count.
+* hikaricp_connections_acquire_seconds_sum: Connection acquire time sum.
+* hikaricp_connections_usage_seconds_max: Connection usage max.
+* hikaricp_connections_usage_seconds_count: Connection usage time count.
+* hikaricp_connections_usage_seconds_sum: Connection usage time sum.
+* hikaricp_connections_max: Max connections.
+* hikaricp_connections_min Min connections
+* hikaricp_connections_active: Active connections.
+* hikaricp_connections_idle: Idle connections.
+* hikaricp_connections_pending: Pending connections.
+* hikaricp_connections_timeout_total: Timeout connections.
+* hikaricp_connections: Total connections
+* jdbc_connections_max: Maximum number of active connections that can be allocated at the same time.
+* jdbc_connections_min: Minimum number of idle connections in the pool.
+* jdbc_connections_idle: Number of established but idle connections.
+* jdbc_connections_active: Current number of active connections that have been allocated from the data source.
+
+### JVM Metrics
+
+* jvm_buffer_total_capacity_bytes: An estimate of the total capacity of the buffers in this pool.
+* jvm_buffer_count_buffers: An estimate of the number of buffers in the pool.
+* jvm_buffer_memory_used_bytes: An estimate of the memory that the Java virtual machine is using for this buffer pool.
+* jvm_memory_committed_bytes: The amount of memory in bytes that is committed for the Java virtual machine to use.
+* jvm_memory_max_bytes: The maximum amount of memory in bytes that can be used for memory management.
+* jvm_memory_used_bytes: The amount of used memory.
+* jvm_threads_peak_threads: The peak live thread count since the Java virtual machine started or peak was reset.
+* jvm_threads_states_threads: The current number of threads having NEW state.
+* jvm_gc_memory_allocated_bytes_total: Incremented for an increase in the size of the (young) heap memory pool after one GC to before the next.
+* jvm_gc_max_data_size_bytes: Max size of long-lived heap memory pool.
+* jvm_gc_pause_seconds_count: Time spent count in GC pause.
+* jvm_gc_pause_seconds_sum: Time spent sum in GC pause.
+* jvm_gc_pause_seconds_max: Time spent max in GC pause.
+* jvm_gc_live_data_size_bytes: Size of long-lived heap memory pool after reclamation.
+* jvm_gc_memory_promoted_bytes_total: Count of positive increases in the size of the old generation memory pool before GC to after GC.
+* jvm_classes_loaded_classes: The number of classes that are currently loaded in the Java virtual machine.
+* jvm_threads_live_threads: The current number of live threads including both daemon and non-daemon threads.
+* jvm_threads_daemon_threads: The current number of live daemon threads.
+* jvm_classes_unloaded_classes_total: The total number of classes unloaded since the Java virtual machine has started execution.
+* process_cpu_usage: The "recent cpu usage" for the Java Virtual Machine process.
+* process_start_time_seconds: Start time of the process since unix epoch.
+* process_uptime_seconds: The uptime of the Java virtual machine.
+
+
+## Other Metrics
+* jetty_threads_config_max: The maximum number of threads in the pool.
+* jetty_threads_config_min: The minimum number of threads in the pool.
+* jetty_threads_current: The total number of threads in the pool.
+* jetty_threads_idle: The number of idle threads in the pool.
+* jetty_threads_busy: The number of busy threads in the pool.
+* jetty_threads_jobs: Number of jobs queued waiting for a thread.
+* process_files_max_files: The maximum file descriptor count.
+* process_files_open_files: The open file descriptor count.
+* system_cpu_usage: The "recent cpu usage" for the whole system.
+* system_cpu_count: The number of processors available to the Java virtual machine.
+* system_load_average_1m: The sum of the number of runnable entities queued to available processors and the number of runnable entities running on the available processors averaged over a period of time.
+* logback_events_total: Number of level events that made it to the logs
\ No newline at end of file
diff --git a/docs/img/metrics/metrics-datasource.png b/docs/img/metrics/metrics-datasource.png
new file mode 100644
index 0000000000..7bc96ddd4e
Binary files /dev/null and b/docs/img/metrics/metrics-datasource.png differ
diff --git a/docs/img/metrics/metrics-master.png b/docs/img/metrics/metrics-master.png
new file mode 100644
index 0000000000..af8c44947e
Binary files /dev/null and b/docs/img/metrics/metrics-master.png differ
diff --git a/docs/img/metrics/metrics-worker.png b/docs/img/metrics/metrics-worker.png
new file mode 100644
index 0000000000..a45fb93f1a
Binary files /dev/null and b/docs/img/metrics/metrics-worker.png differ
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/banner.txt b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/banner.txt
index 1b0e621288..8353e2c083 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/banner.txt
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/banner.txt
@@ -8,5 +8,5 @@ ${AnsiColor.BLUE}${AnsiStyle.BOLD}
               |_|
 ================================================================================
 ${AnsiColor.BLUE}${AnsiStyle.BOLD}
-::  Dolphinscheduler alert server ::  ${application.formatted-version}
+::  DolphinScheduler alert server ::  ${application.formatted-version}
 ${AnsiStyle.NORMAL}
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
index 366457f504..542b8048ed 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -153,6 +153,9 @@ spring:
       on-profile: mysql
   datasource:
     driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+    username: root
+    password: root
   quartz:
     properties:
       org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
diff --git a/dolphinscheduler-api/src/main/resources/banner.txt b/dolphinscheduler-api/src/main/resources/banner.txt
index 28ece5e91f..49aa2a889e 100644
--- a/dolphinscheduler-api/src/main/resources/banner.txt
+++ b/dolphinscheduler-api/src/main/resources/banner.txt
@@ -8,5 +8,5 @@ ${AnsiColor.BLUE}${AnsiStyle.BOLD}
               |_|
 ================================================================================
 ${AnsiColor.BLUE}${AnsiStyle.BOLD}
-::  Dolphinscheduler api server ::  ${application.formatted-version}
+::  DolphinScheduler api server ::  ${application.formatted-version}
 ${AnsiStyle.NORMAL}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index 0382b53c2c..b7714c7817 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -369,22 +369,22 @@ public final class Constants {
     /**
      * sleep 1000ms
      */
-    public static final int SLEEP_TIME_MILLIS = 1000;
+    public static final long SLEEP_TIME_MILLIS = 1_000L;
 
     /**
      * short sleep 100ms
      */
-    public static final int SLEEP_TIME_MILLIS_SHORT = 100;
+    public static final long SLEEP_TIME_MILLIS_SHORT = 100L;
 
     /**
      * one second mils
      */
-    public static final int SECOND_TIME_MILLIS = 1000;
+    public static final long SECOND_TIME_MILLIS = 1_000L;
 
     /**
      * master task instance cache-database refresh interval
      */
-    public static final int CACHE_REFRESH_TIME_MILLIS = 20 * 1000;
+    public static final long CACHE_REFRESH_TIME_MILLIS = 20 * 1_000L;
 
     /**
      * heartbeat for zk info length
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
index 5a12fb6dc7..632dcedd8d 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java
@@ -262,7 +262,7 @@ public class TaskNode {
         this.runFlag = runFlag;
     }
 
-    public Boolean isForbidden() {
+    public boolean isForbidden() {
         return (!StringUtils.isEmpty(this.runFlag)
                 && this.runFlag.equals(Constants.FLOWNODE_RUN_FLAG_FORBIDDEN));
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
index 137bd4ecc5..dc562d37bd 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
@@ -18,11 +18,14 @@
 package org.apache.dolphinscheduler.server.master.cache.impl;
 
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
 import java.util.Collection;
 import java.util.concurrent.ConcurrentHashMap;
 
+import javax.annotation.PostConstruct;
+
 import org.springframework.stereotype.Component;
 
 import com.google.common.collect.ImmutableList;
@@ -35,6 +38,11 @@ public class ProcessInstanceExecCacheManagerImpl implements ProcessInstanceExecC
 
     private final ConcurrentHashMap<Integer, WorkflowExecuteRunnable> processInstanceExecMaps = new ConcurrentHashMap<>();
 
+    @PostConstruct
+    public void registerMetrics() {
+        ProcessInstanceMetrics.registerProcessInstanceRunningGauge(processInstanceExecMaps::size);
+    }
+
     @Override
     public WorkflowExecuteRunnable getByProcessInstanceId(int processInstanceId) {
         return processInstanceExecMaps.get(processInstanceId);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index f9c19c48fd..0dbfb73a2c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.server.master.dispatch.ExecutorDispatcher;
 import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
 import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
 import org.apache.dolphinscheduler.server.master.dispatch.exceptions.ExecuteException;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEvent;
 import org.apache.dolphinscheduler.server.master.processor.queue.TaskEventService;
 import org.apache.dolphinscheduler.service.exceptions.TaskPriorityQueueException;
@@ -53,6 +54,11 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import io.micrometer.core.annotation.Counted;
+import io.micrometer.core.annotation.Timed;
+
+import org.apache.commons.lang3.time.StopWatch;
+
 /**
  * TaskUpdateQueue consumer
  */
@@ -119,6 +125,7 @@ public class TaskPriorityQueueConsumer extends Thread {
                 List<TaskPriority> failedDispatchTasks = this.batchDispatch(fetchTaskNum);
 
                 if (!failedDispatchTasks.isEmpty()) {
+                    TaskMetrics.incTaskDispatchFailed(failedDispatchTasks.size());
                     for (TaskPriority dispatchFailedTask : failedDispatchTasks) {
                         taskPriorityQueue.put(dispatchFailedTask);
                     }
@@ -129,6 +136,7 @@ public class TaskPriorityQueueConsumer extends Thread {
                     }
                 }
             } catch (Exception e) {
+                TaskMetrics.incTaskDispatchError();
                 logger.error("dispatcher task error", e);
             }
         }
@@ -137,7 +145,7 @@ public class TaskPriorityQueueConsumer extends Thread {
     /**
      * batch dispatch with thread pool
      */
-    private List<TaskPriority> batchDispatch(int fetchTaskNum) throws TaskPriorityQueueException, InterruptedException {
+    public List<TaskPriority> batchDispatch(int fetchTaskNum) throws TaskPriorityQueueException, InterruptedException {
         List<TaskPriority> failedDispatchTasks = Collections.synchronizedList(new ArrayList<>());
         CountDownLatch latch = new CountDownLatch(fetchTaskNum);
 
@@ -169,6 +177,7 @@ public class TaskPriorityQueueConsumer extends Thread {
      * @return result
      */
     protected boolean dispatchTask(TaskPriority taskPriority) {
+        TaskMetrics.incTaskDispatch();
         boolean result = false;
         try {
             TaskExecutionContext context = taskPriority.getTaskExecutionContext();
@@ -217,7 +226,7 @@ public class TaskPriorityQueueConsumer extends Thread {
      * @param taskInstanceId taskInstanceId
      * @return taskInstance is final state
      */
-    public Boolean taskInstanceIsFinalState(int taskInstanceId) {
+    public boolean taskInstanceIsFinalState(int taskInstanceId) {
         TaskInstance taskInstance = processService.findTaskInstanceById(taskInstanceId);
         return taskInstance.getState().typeIsFinished();
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java
new file mode 100644
index 0000000000..e5c86d9641
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/MasterServerMetrics.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.metrics;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+
+public final class MasterServerMetrics {
+
+    private MasterServerMetrics() {
+        throw new UnsupportedOperationException("Utility class");
+    }
+
+    /**
+     * Used to measure the master server is overload.
+     */
+    private static final Counter MASTER_OVERLOAD_COUNTER =
+            Counter.builder("dolphinscheduler_master_overload_count")
+                    .description("Master server overload count")
+                    .register(Metrics.globalRegistry);
+
+    /**
+     * Used to measure the number of process command consumed by master.
+     */
+    private static final Counter MASTER_CONSUME_COMMAND_COUNTER =
+            Counter.builder("dolphinscheduler_master_consume_command_count")
+                    .description("Master server consume command count")
+                    .register(Metrics.globalRegistry);
+
+    public static void incMasterOverload() {
+        MASTER_OVERLOAD_COUNTER.increment();
+    }
+
+    public static void incMasterConsumeCommand(int commandCount) {
+        MASTER_CONSUME_COMMAND_COUNTER.increment(commandCount);
+    }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
new file mode 100644
index 0000000000..9e9b11c95d
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.metrics;
+
+import java.util.function.Supplier;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.Metrics;
+
+public final class ProcessInstanceMetrics {
+
+    private ProcessInstanceMetrics() {
+        throw new UnsupportedOperationException("Utility class");
+    }
+
+    private static final Counter PROCESS_INSTANCE_SUBMIT_COUNTER =
+            Counter.builder("dolphinscheduler_process_instance_submit_count")
+                    .description("Process instance submit total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter PROCESS_INSTANCE_TIMEOUT_COUNTER =
+            Counter.builder("dolphinscheduler_process_instance_timeout_count")
+                    .description("Process instance timeout total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter PROCESS_INSTANCE_FINISH_COUNTER =
+            Counter.builder("dolphinscheduler_process_instance_finish_count")
+                    .description("Process instance finish total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter PROCESS_INSTANCE_SUCCESS_COUNTER =
+            Counter.builder("dolphinscheduler_process_instance_success_count")
+                    .description("Process instance success total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter PROCESS_INSTANCE_FAILURE_COUNTER =
+            Counter.builder("dolphinscheduler_process_instance_failure_count")
+                    .description("Process instance failure total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter PROCESS_INSTANCE_STOP_COUNTER =
+            Counter.builder("dolphinscheduler_process_instance_stop_count")
+                    .description("Process instance stop total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter PROCESS_INSTANCE_FAILOVER_COUNTER =
+            Counter.builder("dolphinscheduler_process_instance_failover_count")
+                    .description("Process instance failover total count")
+                    .register(Metrics.globalRegistry);
+
+    public static synchronized void registerProcessInstanceRunningGauge(Supplier<Number> function) {
+        Gauge.builder("dolphinscheduler_process_instance_running_gauge", function)
+                .description("The current running process instance count")
+                .register(Metrics.globalRegistry);
+    }
+
+    public static void incProcessInstanceSubmit() {
+        PROCESS_INSTANCE_SUBMIT_COUNTER.increment();
+    }
+
+    public static void incProcessInstanceTimeout() {
+        PROCESS_INSTANCE_TIMEOUT_COUNTER.increment();
+    }
+
+    public static void incProcessInstanceFinish() {
+        PROCESS_INSTANCE_FINISH_COUNTER.increment();
+    }
+
+    public static void incProcessInstanceSuccess() {
+        PROCESS_INSTANCE_SUCCESS_COUNTER.increment();
+    }
+
+    public static void incProcessInstanceFailure() {
+        PROCESS_INSTANCE_FAILURE_COUNTER.increment();
+    }
+
+    public static void incProcessInstanceStop() {
+        PROCESS_INSTANCE_STOP_COUNTER.increment();
+    }
+
+    public static void incProcessInstanceFailover() {
+        PROCESS_INSTANCE_FAILOVER_COUNTER.increment();
+    }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java
new file mode 100644
index 0000000000..c4516d0644
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.metrics;
+
+import java.util.function.Supplier;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.Metrics;
+
+
+public final class TaskMetrics {
+    private TaskMetrics() {
+        throw new UnsupportedOperationException("Utility class");
+    }
+
+    private static final Counter TASK_SUBMIT_COUNTER =
+            Counter.builder("dolphinscheduler_task_submit_count")
+                    .description("Task submit total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_FINISH_COUNTER =
+            Counter.builder("dolphinscheduler_task_finish_count")
+                    .description("Task finish total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_SUCCESS_COUNTER =
+            Counter.builder("dolphinscheduler_task_success_count")
+                    .description("Task success total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_FAILURE_COUNTER =
+            Counter.builder("dolphinscheduler_task_failure_count")
+                    .description("Task failure total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_TIMEOUT_COUNTER =
+            Counter.builder("dolphinscheduler_task_timeout_count")
+                    .description("Task timeout total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_RETRY_COUNTER =
+            Counter.builder("dolphinscheduler_task_retry_count")
+                    .description("Task retry total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_STOP_COUNTER =
+            Counter.builder("dolphinscheduler_task_stop_count")
+                    .description("Task stop total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_FAILOVER_COUNTER =
+            Counter.builder("dolphinscheduler_task_failover_count")
+                    .description("Task failover total count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_DISPATCH_COUNTER =
+            Counter.builder("dolphinscheduler_task_dispatch_count")
+                    .description("Task dispatch count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_DISPATCHER_FAILED =
+            Counter.builder("dolphinscheduler_task_dispatch_failed_count")
+                    .description("Task dispatch failed count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter TASK_DISPATCH_ERROR =
+            Counter.builder("dolphinscheduler_task_dispatch_error_count")
+                    .description("Task dispatch error")
+                    .register(Metrics.globalRegistry);
+
+    public static void incTaskSubmit() {
+        TASK_SUBMIT_COUNTER.increment();
+    }
+
+    public synchronized static void registerTaskRunning(Supplier<Number> consumer) {
+        Gauge.builder("dolphinscheduler_task_running_gauge", consumer)
+                .description("Task running count")
+                .register(Metrics.globalRegistry);
+    }
+
+    public static void incTaskFinish() {
+        TASK_FINISH_COUNTER.increment();
+    }
+
+    public static void incTaskSuccess() {
+        TASK_SUCCESS_COUNTER.increment();
+    }
+
+    public static void incTaskFailure() {
+        TASK_FAILURE_COUNTER.increment();
+    }
+
+    public static void incTaskTimeout() {
+        TASK_TIMEOUT_COUNTER.increment();
+    }
+
+    public static void incTaskRetry() {
+        TASK_RETRY_COUNTER.increment();
+    }
+
+    public static void incTaskStop() {
+        TASK_STOP_COUNTER.increment();
+    }
+
+    public static void incTaskFailover() {
+        TASK_FAILOVER_COUNTER.increment();
+    }
+
+    public static void incTaskDispatchFailed(int failedCount) {
+        TASK_DISPATCHER_FAILED.increment(failedCount);
+    }
+
+    public static void incTaskDispatchError() {
+        TASK_DISPATCH_ERROR.increment();
+    }
+
+    public static void incTaskDispatch() {
+        TASK_DISPATCH_COUNTER.increment();
+    }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
index 5fd812f162..4bac6cdf15 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
@@ -62,7 +62,7 @@ public class FailoverExecuteThread extends Thread {
             } catch (Exception e) {
                 logger.error("failover execute error", e);
             } finally {
-                ThreadUtils.sleep((long) Constants.SLEEP_TIME_MILLIS * masterConfig.getFailoverInterval() * 60);
+                ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * masterConfig.getFailoverInterval() * 60);
             }
         }
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index e688863d06..ae9a461d10 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.SlotCheckState;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.dao.entity.Command;
@@ -31,6 +30,7 @@ import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
+import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
 import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
 import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
 import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -133,6 +133,7 @@ public class MasterSchedulerService extends Thread {
             try {
                 boolean runCheckFlag = OSUtils.checkResource(masterConfig.getMaxCpuLoadAvg(), masterConfig.getReservedMemory());
                 if (!runCheckFlag) {
+                    MasterServerMetrics.incMasterOverload();
                     Thread.sleep(Constants.SLEEP_TIME_MILLIS);
                     continue;
                 }
@@ -159,6 +160,7 @@ public class MasterSchedulerService extends Thread {
         if (CollectionUtils.isEmpty(processInstances)) {
             return;
         }
+        MasterServerMetrics.incMasterConsumeCommand(commands.size());
 
         for (ProcessInstance processInstance : processInstances) {
             if (processInstance == null) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 67bfe3c928..82f942e03b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -66,6 +66,8 @@ import org.apache.dolphinscheduler.remote.command.HostUpdateCommand;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.master.runner.task.ITaskProcessor;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskAction;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
@@ -83,6 +85,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -92,7 +95,6 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -249,6 +251,7 @@ public class WorkflowExecuteRunnable implements Runnable {
         this.nettyExecutorManager = nettyExecutorManager;
         this.processAlertManager = processAlertManager;
         this.stateWheelExecuteThread = stateWheelExecuteThread;
+        TaskMetrics.registerTaskRunning(readyToSubmitTaskQueue::size);
     }
 
     /**
@@ -291,7 +294,7 @@ public class WorkflowExecuteRunnable implements Runnable {
 
     public boolean addStateEvent(StateEvent stateEvent) {
         if (processInstance.getId() != stateEvent.getProcessInstanceId()) {
-            logger.info("state event would be abounded :{}", stateEvent.toString());
+            logger.info("state event would be abounded :{}", stateEvent);
             return false;
         }
         this.stateEvents.add(stateEvent);
@@ -307,7 +310,7 @@ public class WorkflowExecuteRunnable implements Runnable {
     }
 
     private boolean stateEventHandler(StateEvent stateEvent) {
-        logger.info("process event: {}", stateEvent.toString());
+        logger.info("process event: {}", stateEvent);
 
         if (!checkProcessInstance(stateEvent)) {
             return false;
@@ -316,21 +319,26 @@ public class WorkflowExecuteRunnable implements Runnable {
         boolean result = false;
         switch (stateEvent.getType()) {
             case PROCESS_STATE_CHANGE:
+                measureProcessState(stateEvent);
                 result = processStateChangeHandler(stateEvent);
                 break;
             case TASK_STATE_CHANGE:
+                measureTaskState(stateEvent);
                 result = taskStateChangeHandler(stateEvent);
                 break;
             case PROCESS_TIMEOUT:
+                ProcessInstanceMetrics.incProcessInstanceTimeout();
                 result = processTimeout();
                 break;
             case TASK_TIMEOUT:
+                TaskMetrics.incTaskTimeout();
                 result = taskTimeout(stateEvent);
                 break;
             case WAIT_TASK_GROUP:
                 result = checkForceStartAndWakeUp(stateEvent);
                 break;
             case TASK_RETRY:
+                TaskMetrics.incTaskRetry();
                 result = taskRetryEventHandler(stateEvent);
                 break;
             case PROCESS_BLOCKED:
@@ -437,10 +445,10 @@ public class WorkflowExecuteRunnable implements Runnable {
 
     private void taskFinished(TaskInstance taskInstance) {
         logger.info("work flow {} task id:{} code:{} state:{} ",
-                    processInstance.getId(),
-                    taskInstance.getId(),
-                    taskInstance.getTaskCode(),
-                    taskInstance.getState());
+                processInstance.getId(),
+                taskInstance.getId(),
+                taskInstance.getTaskCode(),
+                taskInstance.getState());
 
         activeTaskProcessorMaps.remove(taskInstance.getTaskCode());
         stateWheelExecuteThread.removeTask4TimeoutCheck(processInstance, taskInstance);
@@ -734,7 +742,7 @@ public class WorkflowExecuteRunnable implements Runnable {
             scheduleDate = complementListDate.get(0);
         } else if (processInstance.getState().typeIsFinished()) {
             endProcess();
-            if (complementListDate.size() <= 0) {
+            if (complementListDate.isEmpty()) {
                 logger.info("process complement end. process id:{}", processInstance.getId());
                 return true;
             }
@@ -745,9 +753,9 @@ public class WorkflowExecuteRunnable implements Runnable {
                 return true;
             }
             logger.info("process complement continue. process id:{}, schedule time:{} complementListDate:{}",
-                        processInstance.getId(),
-                        processInstance.getScheduleTime(),
-                        complementListDate.toString());
+                processInstance.getId(),
+                processInstance.getScheduleTime(),
+                complementListDate.toString());
             scheduleDate = complementListDate.get(index + 1);
         }
         //the next process complement
@@ -948,7 +956,7 @@ public class WorkflowExecuteRunnable implements Runnable {
             }
         }
 
-        if (processInstance.isComplementData() && complementListDate.size() == 0) {
+        if (processInstance.isComplementData() && complementListDate.isEmpty()) {
             Map<String, String> cmdParam = JSONUtils.toMap(processInstance.getCommandParam());
             if (cmdParam != null && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) {
                 // reset global params while there are start parameters
@@ -957,19 +965,17 @@ public class WorkflowExecuteRunnable implements Runnable {
                 Date start = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
                 Date end = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
                 List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(processInstance.getProcessDefinitionCode());
-                if (complementListDate.size() == 0 && needComplementProcess()) {
+                if (complementListDate.isEmpty() && needComplementProcess()) {
                     complementListDate = CronUtils.getSelfFireDateList(start, end, schedules);
                     logger.info(" process definition code:{} complement data: {}",
-                                processInstance.getProcessDefinitionCode(),
-                                complementListDate.toString());
+                        processInstance.getProcessDefinitionCode(), complementListDate.toString());
 
-                    if (complementListDate.size() > 0 && Flag.NO == processInstance.getIsSubProcess()) {
+                    if (!complementListDate.isEmpty() && Flag.NO == processInstance.getIsSubProcess()) {
                         processInstance.setScheduleTime(complementListDate.get(0));
-                        processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(processDefinition.getGlobalParamMap(),
-                                                                                          processDefinition.getGlobalParamList(),
-                                                                                          CommandType.COMPLEMENT_DATA,
-                                                                                          processInstance.getScheduleTime(),
-                                                                                          cmdParam.get(Constants.SCHEDULE_TIMEZONE)));
+                        processInstance.setGlobalParams(ParameterUtils.curingGlobalParams(
+                            processDefinition.getGlobalParamMap(),
+                            processDefinition.getGlobalParamList(),
+                            CommandType.COMPLEMENT_DATA, processInstance.getScheduleTime(), cmdParam.get(Constants.SCHEDULE_TIMEZONE)));
                         processService.updateProcessInstance(processInstance);
                     }
                 }
@@ -1322,12 +1328,10 @@ public class WorkflowExecuteRunnable implements Runnable {
             TaskInstance endTaskInstance = taskInstanceMap.get(completeTaskMap.get(NumberUtils.toLong(parentNodeCode)));
             String taskInstanceVarPool = endTaskInstance.getVarPool();
             if (StringUtils.isNotEmpty(taskInstanceVarPool)) {
-                Set<Property> taskProperties = JSONUtils.toList(taskInstanceVarPool, Property.class)
-                    .stream().collect(Collectors.toSet());
+                Set<Property> taskProperties = new HashSet<>(JSONUtils.toList(taskInstanceVarPool, Property.class));
                 String processInstanceVarPool = processInstance.getVarPool();
                 if (StringUtils.isNotEmpty(processInstanceVarPool)) {
-                    Set<Property> properties = JSONUtils.toList(processInstanceVarPool, Property.class)
-                        .stream().collect(Collectors.toSet());
+                    Set<Property> properties = new HashSet<>(JSONUtils.toList(processInstanceVarPool, Property.class));
                     properties.addAll(taskProperties);
                     processInstance.setVarPool(JSONUtils.toJsonString(properties));
                 } else {
@@ -1733,9 +1737,7 @@ public class WorkflowExecuteRunnable implements Runnable {
                 return;
             }
             logger.info("add task to stand by list, task name:{}, task id:{}, task code:{}",
-                        taskInstance.getName(),
-                        taskInstance.getId(),
-                        taskInstance.getTaskCode());
+                taskInstance.getName(), taskInstance.getId(), taskInstance.getTaskCode());
             readyToSubmitTaskQueue.put(taskInstance);
         } catch (Exception e) {
             logger.error("add task instance to readyToSubmitTaskQueue, taskName:{}, task id:{}", taskInstance.getName(), taskInstance.getId(), e);
@@ -2028,12 +2030,26 @@ public class WorkflowExecuteRunnable implements Runnable {
         }
     }
 
-    private void measureTaskState(StateEvent taskStateEvent) {
-        if (taskStateEvent == null || taskStateEvent.getExecutionStatus() == null) {
-            // the event is broken
-            logger.warn("The task event is broken..., taskEvent: {}", taskStateEvent);
-            return;
+    private void measureProcessState(StateEvent processStateEvent) {
+        if (processStateEvent.getExecutionStatus().typeIsFinished()) {
+            ProcessInstanceMetrics.incProcessInstanceFinish();
+        }
+        switch (processStateEvent.getExecutionStatus()) {
+            case STOP:
+                ProcessInstanceMetrics.incProcessInstanceStop();
+                break;
+            case SUCCESS:
+                ProcessInstanceMetrics.incProcessInstanceSuccess();
+                break;
+            case FAILURE:
+                ProcessInstanceMetrics.incProcessInstanceFailure();
+                break;
+            default:
+                break;
         }
+    }
+
+    private void measureTaskState(StateEvent taskStateEvent) {
         if (taskStateEvent.getExecutionStatus().typeIsFinished()) {
             TaskMetrics.incTaskFinish();
         }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index c3d771e365..36edf1371b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
 import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.service.process.ProcessService;
 
 import org.apache.commons.lang.StringUtils;
@@ -45,6 +46,8 @@ import org.springframework.stereotype.Component;
 import org.springframework.util.concurrent.ListenableFuture;
 import org.springframework.util.concurrent.ListenableFutureCallback;
 
+import com.google.common.base.Strings;
+
 @Component
 public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
 
@@ -94,6 +97,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
      * start workflow
      */
     public void startWorkflow(WorkflowExecuteRunnable workflowExecuteThread) {
+        ProcessInstanceMetrics.incProcessInstanceSubmit();
         submit(workflowExecuteThread);
     }
 
@@ -109,7 +113,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
         }
         multiThreadFilterMap.put(workflowExecuteThread.getKey(), workflowExecuteThread);
         int processInstanceId = workflowExecuteThread.getProcessInstance().getId();
-        ListenableFuture future = this.submitListenable(workflowExecuteThread::handleEvents);
+        ListenableFuture<?> future = this.submitListenable(workflowExecuteThread::handleEvents);
         future.addCallback(new ListenableFutureCallback() {
             @Override
             public void onFailure(Throwable ex) {
@@ -120,7 +124,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
             @Override
             public void onSuccess(Object result) {
                 // if an exception occurs, first, the error message cannot be printed in the log;
-                // secondly, the `multiThreadFilterMap` cannot be remove the `workflowExecuteThread`, resulting in the state of process instance cannot be changed and memory leak
+                // secondly, the `multiThreadFilterMap` cannot remove the `workflowExecuteThread`, resulting in the state of process instance cannot be changed and memory leak
                 try {
                     if (workflowExecuteThread.workFlowFinish()) {
                         stateWheelExecuteThread.removeProcess4TimeoutCheck(workflowExecuteThread.getProcessInstance());
@@ -144,12 +148,14 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
             return;
         }
         Map<ProcessInstance, TaskInstance> fatherMaps = processService.notifyProcessList(finishProcessInstance.getId());
-        for (ProcessInstance processInstance : fatherMaps.keySet()) {
+        for (Map.Entry<ProcessInstance, TaskInstance> entry : fatherMaps.entrySet()) {
+            ProcessInstance processInstance = entry.getKey();
+            TaskInstance taskInstance = entry.getValue();
             String address = NetUtils.getAddr(masterConfig.getListenPort());
             if (processInstance.getHost().equalsIgnoreCase(address)) {
-                this.notifyMyself(processInstance, fatherMaps.get(processInstance));
+                this.notifyMyself(processInstance, taskInstance);
             } else {
-                this.notifyProcess(finishProcessInstance, processInstance, fatherMaps.get(processInstance));
+                this.notifyProcess(finishProcessInstance, processInstance, taskInstance);
             }
         }
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
index 81e3070089..ae8198ebfd 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/CommonTaskProcessor.java
@@ -43,7 +43,7 @@ import com.google.auto.service.AutoService;
 @AutoService(ITaskProcessor.class)
 public class CommonTaskProcessor extends BaseTaskProcessor {
 
-    private TaskPriorityQueue taskUpdateQueue;
+    private TaskPriorityQueue<TaskPriority> taskUpdateQueue;
 
     private NettyExecutorManager nettyExecutorManager = SpringApplicationContext.getBean(NettyExecutorManager.class);
 
@@ -101,7 +101,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
                 this.initQueue();
             }
             if (taskInstance.getState().typeIsFinished()) {
-                logger.info(String.format("submit task , but task [%s] state [%s] is already  finished. ", taskInstance.getName(), taskInstance.getState().toString()));
+                logger.info("submit task , but task [{}] state [{}] is already  finished. ", taskInstance.getName(), taskInstance.getState());
                 return true;
             }
             // task cannot be submitted because its execution state is RUNNING or DELAY.
@@ -125,7 +125,7 @@ public class CommonTaskProcessor extends BaseTaskProcessor {
             taskPriority.setTaskExecutionContext(taskExecutionContext);
 
             taskUpdateQueue.put(taskPriority);
-            logger.info("Master submit task to priority queue success, taskInstanceId : {}", taskInstance.getId());
+            logger.info("master submit success, task : {}", taskInstance.getName());
             return true;
         } catch (Exception e) {
             logger.error("submit task error", e);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
index d5bd131a10..41c2bd56d3 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
 /**
  * the factory to create task processor
  */
-public class TaskProcessorFactory {
+public final class TaskProcessorFactory {
 
     private static final Logger logger = LoggerFactory.getLogger(TaskProcessorFactory.class);
 
@@ -66,10 +66,15 @@ public class TaskProcessorFactory {
 
     /**
      * if match master processor, then this task type is processed on the master
+     *
      * @param type
      * @return
      */
     public static boolean isMasterTask(String type) {
         return PROCESS_MAP.containsKey(type);
     }
+
+    private TaskProcessorFactory() {
+        throw new UnsupportedOperationException("TaskProcessorFactory cannot be instantiated");
+    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
index dad2dbfe1e..3cc92fb905 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
@@ -32,6 +32,8 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
+import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskProcessorFactory;
 import org.apache.dolphinscheduler.server.utils.ProcessUtils;
@@ -157,6 +159,7 @@ public class FailoverService {
             }
 
             LOGGER.info("failover process instance id: {}", processInstance.getId());
+            ProcessInstanceMetrics.incProcessInstanceFailover();
             //updateProcessInstance host is null and insert into command
             processInstance.setHost(Constants.NULL);
             processService.processNeedFailoverProcessInstances(processInstance);
@@ -227,7 +230,7 @@ public class FailoverService {
         if (!checkTaskInstanceNeedFailover(servers, taskInstance)) {
             return;
         }
-
+        TaskMetrics.incTaskFailover();
         boolean isMasterTask = TaskProcessorFactory.isMasterTask(taskInstance.getTaskType());
 
         taskInstance.setProcessInstance(processInstance);
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index e353bb8dcc..a6c70f9871 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -136,6 +136,9 @@ spring:
       on-profile: mysql
   datasource:
     driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+    username: root
+    password: root
   quartz:
     properties:
       org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate
diff --git a/dolphinscheduler-master/src/main/resources/banner.txt b/dolphinscheduler-master/src/main/resources/banner.txt
index ce8c3b508d..ef081edfff 100644
--- a/dolphinscheduler-master/src/main/resources/banner.txt
+++ b/dolphinscheduler-master/src/main/resources/banner.txt
@@ -8,5 +8,5 @@ ${AnsiColor.BLUE}${AnsiStyle.BOLD}
               |_|
 ================================================================================
 ${AnsiColor.BLUE}${AnsiStyle.BOLD}
-::  Dolphinscheduler master server ::  ${application.formatted-version}
+::  DolphinScheduler master server ::  ${application.formatted-version}
 ${AnsiStyle.NORMAL}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
index 339c5e80ae..f2d263f699 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/WorkflowExecuteTaskTest.java
@@ -84,8 +84,6 @@ public class WorkflowExecuteTaskTest {
 
     private ApplicationContext applicationContext;
 
-    private TaskProcessorFactory taskProcessorFactory;
-
     private StateWheelExecuteThread stateWheelExecuteThread;
 
     @Before
@@ -100,8 +98,6 @@ public class WorkflowExecuteTaskTest {
         processService = mock(ProcessService.class);
         Mockito.when(applicationContext.getBean(ProcessService.class)).thenReturn(processService);
 
-        taskProcessorFactory = mock(TaskProcessorFactory.class);
-
         processInstance = mock(ProcessInstance.class);
         Mockito.when(processInstance.getState()).thenReturn(ExecutionStatus.SUCCESS);
         Mockito.when(processInstance.getHistoryCmd()).thenReturn(CommandType.COMPLEMENT_DATA.toString());
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java
deleted file mode 100644
index 42b49888bc..0000000000
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManagerTest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.registry;
-
-import org.apache.dolphinscheduler.dao.AlertDao;
-import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
-import org.apache.dolphinscheduler.service.registry.RegistryClient;
-
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-/**
- * server node manager test
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ RegistryClient.class })
-@PowerMockIgnore({"javax.management.*"})
-public class ServerNodeManagerTest {
-
-    private ServerNodeManager serverNodeManager;
-
-    @Mock
-    private WorkerGroupMapper workerGroupMapper;
-
-    @Mock
-    private AlertDao alertDao;
-
-    @Before
-    public void before() {
-        PowerMockito.suppress(PowerMockito.constructor(RegistryClient.class));
-        serverNodeManager = PowerMockito.mock(ServerNodeManager.class);
-    }
-
-    @Test
-    public void test(){
-        //serverNodeManager.getWorkerGroupNodes()
-    }
-
-}
diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java
index d8b7297853..b7aecf7ee6 100644
--- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java
+++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java
@@ -20,6 +20,10 @@
 
 package org.apache.dolphinscheduler.meter;
 
+import javax.annotation.PostConstruct;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
@@ -30,6 +34,18 @@ import io.micrometer.core.aop.CountedAspect;
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
 
+/**
+ * This configuration class is used to config the metrics. We use <a href="https://micrometer.io/docs/concepts">micrometer</a> as the metrics fade.
+ *
+ * <p>To open the metrics, you need to set the property "metrics.enabled" to true. Right now, we only support expose the metrics to Prometheus,
+ * after you open metrics expose, you can get the metrics data at: http://host:port/actuator/prometheus.
+ * <p>You can use the below method to get a meter:
+ * <pre>
+ *     {@code
+ *      Counter counter = Metrics.counter("name", "tag1", "tag2");
+ *     }
+ * </pre>
+ */
 @Configuration
 @EnableAspectJAutoProxy
 @EnableAutoConfiguration
diff --git a/dolphinscheduler-meter/src/main/resources/grafana-demo/docker-compose.yaml b/dolphinscheduler-meter/src/main/resources/grafana-demo/docker-compose.yaml
index 6705f30644..185803ad4f 100644
--- a/dolphinscheduler-meter/src/main/resources/grafana-demo/docker-compose.yaml
+++ b/dolphinscheduler-meter/src/main/resources/grafana-demo/docker-compose.yaml
@@ -32,9 +32,10 @@ services:
     image: grafana/grafana
     networks: [ test ]
     ports:
-      - "3000:3000"
+      # due to the DolphinScheduler frontend port is 3000, so we change the grafana default port to 3001.
+      - "3001:3000"
     environment:
-      GF_AUTH_ANONYMOUS_ENABLED: true
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
     volumes:
       - ../grafana:/dashboards:ro
       - ./datasources:/etc/grafana/provisioning/datasources:ro
diff --git a/dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerMaster.json b/dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerMaster.json
index 42b17494c0..90ba34c5fc 100644
--- a/dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerMaster.json
+++ b/dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerMaster.json
@@ -3,7 +3,10 @@
     "list": [
       {
         "builtIn": 1,
-        "datasource": "-- Grafana --",
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
@@ -19,7 +22,10 @@
         "type": "dashboard"
       },
       {
-        "datasource": "Prometheus",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
         "enable": true,
         "expr": "resets(process_uptime_seconds{application=\"$application\", instance=\"$instance\"}[1m]) > 0",
         "iconColor": "rgba(255, 96, 96, 1)",
@@ -37,32 +43,1506 @@
   "fiscalYearStartMonth": 0,
   "gnetId": 4701,
   "graphTooltip": 1,
-  "id": 4,
-  "iteration": 1636680427082,
+  "iteration": 1654674717443,
   "links": [],
   "liveNow": false,
   "panels": [
     {
-      "collapsed": false,
-      "datasource": null,
+      "collapsed": true,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
         "y": 0
       },
+      "id": 164,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 0,
+            "y": 1
+          },
+          "id": 148,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "increase(dolphinscheduler_master_overload_count_total[1m])",
+              "legendFormat": "",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Master Overload/1m",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 6,
+            "y": 1
+          },
+          "id": 150,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "increase(dolphinscheduler_master_consume_command_count_total{}[1m])",
+              "legendFormat": "master_consume_command",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Master Consume Command/1m",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 12,
+            "y": 1
+          },
+          "id": 168,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "exemplar": false,
+              "expr": "jvm_threads_live_threads{application=\"master-server\"}",
+              "legendFormat": "live_thread",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "jvm_threads_daemon_threads{application=\"master-server\"}",
+              "hide": false,
+              "legendFormat": "daemon_thread",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "editorMode": "code",
+              "expr": "jvm_threads_peak_threads{application=\"master-server\"}",
+              "hide": false,
+              "legendFormat": "peak_thread",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "JVM Thread",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 18,
+            "y": 1
+          },
+          "id": 170,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "expr": "jvm_threads_states_threads{application=\"master-server\"}",
+              "refId": "A"
+            }
+          ],
+          "title": "Thread Status",
+          "type": "timeseries"
+        }
+      ],
+      "title": "MasterServer",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "datasource": {
+        "type": "datasource",
+        "uid": "grafana"
+      },
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 1
+      },
       "id": 126,
-      "panels": [],
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "JOBS"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 2
+          },
+          "id": 63,
+          "links": [],
+          "maxDataPoints": 100,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "auto",
+            "orientation": "horizontal",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "text": {},
+            "textMode": "auto"
+          },
+          "pluginVersion": "8.5.3",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(quartz_job_executed_total)",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "",
+              "metric": "",
+              "refId": "A",
+              "step": 14400
+            }
+          ],
+          "title": "Job Total Count",
+          "type": "stat"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "red",
+                    "value": null
+                  },
+                  {
+                    "color": "green",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percent"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 2
+          },
+          "id": 144,
+          "links": [],
+          "maxDataPoints": 100,
+          "options": {
+            "orientation": "horizontal",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showThresholdLabels": false,
+            "showThresholdMarkers": true,
+            "text": {}
+          },
+          "pluginVersion": "8.5.3",
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(quartz_job_executed_total{result=\"success\"}) / sum(quartz_job_executed_total) * 100",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "",
+              "metric": "",
+              "refId": "A",
+              "step": 14400
+            }
+          ],
+          "title": "Job Successful Rate",
+          "type": "gauge"
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 6,
+            "x": 0,
+            "y": 10
+          },
+          "hiddenSeries": false,
+          "id": 139,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.3",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(quartz_job_executed_total{})",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "Total",
+              "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "quartz_job_executed_total{result=\"success\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "Successful",
+              "refId": "B"
+            },
+            {
+              "exemplar": true,
+              "expr": "quartz_job_executed_total{result=\"failure\"}",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "Failed ({{exception}})",
+              "refId": "C"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Quartz Job Executed Count",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:1516",
+              "format": "short",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "$$hashKey": "object:1517",
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 6,
+            "x": 6,
+            "y": 10
+          },
+          "hiddenSeries": false,
+          "id": 101,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.3",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "rate(quartz_job_execution_seconds_sum[1m])/rate(quartz_job_execution_seconds_count[1m])",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "avg ({{exception}})",
+              "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "quartz_job_execution_seconds_max",
+              "format": "time_series",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "max ({{exception}})",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Quartz Job Execution Time",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:1671",
+              "format": "s",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "$$hashKey": "object:1672",
+              "format": "short",
+              "label": "",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "description": "",
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 6,
+            "x": 12,
+            "y": 10
+          },
+          "hiddenSeries": false,
+          "id": 119,
+          "legend": {
+            "alignAsTable": false,
+            "avg": false,
+            "current": true,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.5.3",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "sum(increase(quartz_job_executed_total[1m]))",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "Total",
+              "refId": "A"
+            },
+            {
+              "exemplar": true,
+              "expr": "increase(quartz_job_executed_total{result=\"success\"}[1m])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "Successful",
+              "refId": "B"
+            },
+            {
+              "exemplar": true,
+              "expr": "increase(quartz_job_executed_total{result=\"failure\"}[1m])",
+              "format": "time_series",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "Failed ({{exception}})",
+              "refId": "C"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "Quartz Job Executed Count / Minute",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:1516",
+              "format": "short",
+              "logBase": 1,
+              "min": "0",
+              "show": true
+            },
+            {
+              "$$hashKey": "object:1517",
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "cards": {},
+          "color": {
+            "cardColor": "#F2495C",
+            "colorScale": "sqrt",
+            "colorScheme": "interpolateReds",
+            "exponent": 0.5,
+            "mode": "opacity"
+          },
+          "dataFormat": "timeseries",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 6,
+            "x": 18,
+            "y": 10
+          },
+          "heatmap": {},
+          "hideZeroBuckets": false,
+          "highlightCards": true,
+          "id": 146,
+          "legend": {
+            "show": true
+          },
+          "pluginVersion": "8.2.3",
+          "reverseYBuckets": false,
+          "targets": [
+            {
+              "exemplar": true,
+              "expr": "histogram_quantile(0.95, sum(rate(quartz_job_execution_seconds_bucket[5m])) by (le))",
+              "interval": "",
+              "legendFormat": "",
+              "refId": "A"
+            }
+          ],
+          "title": "Quartz Job Execution Time Distribution",
+          "tooltip": {
+            "show": true,
+            "showHistogram": false
+          },
+          "type": "heatmap",
+          "xAxis": {
+            "show": true
+          },
+          "yAxis": {
+            "format": "s",
+            "logBase": 1,
+            "show": true
+          },
+          "yBucketBound": "auto"
+        }
+      ],
       "title": "Scheduler",
       "type": "row"
     },
     {
-      "cacheTimeout": null,
-      "datasource": "Prometheus",
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 2
+      },
+      "id": 166,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 3
+          },
+          "id": 152,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "expr": "increase(dolphinscheduler_process_instance_submit_count_total{}[1m])",
+              "refId": "A"
+            }
+          ],
+          "title": "Process Instance Submit/1m",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 3
+          },
+          "id": 162,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "expr": "increase(dolphinscheduler_process_instance_finish_count_total{}[1m])",
+              "refId": "A"
+            }
+          ],
+          "title": "Process Instance Finish/1m",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 0,
+            "y": 11
+          },
+          "id": 156,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "expr": "increase(dolphinscheduler_process_instance_success_count_total{}[1m])",
+              "refId": "A"
+            }
+          ],
+          "title": "Process Instance Success /1m",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 6,
+            "y": 11
+          },
+          "id": 160,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "expr": "increase(dolphinscheduler_process_instance_stop_count_total{}[1m])",
+              "refId": "A"
+            }
+          ],
+          "title": "Process Instance Stop/1m",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 12,
+            "y": 11
+          },
+          "id": 154,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "expr": "increase(dolphinscheduler_process_instance_timeout_count_total{}[1m])",
+              "refId": "A"
+            }
+          ],
+          "title": "Process Instance Timeout/1m",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 6,
+            "x": 18,
+            "y": 11
+          },
+          "id": 158,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom"
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "PBFA97CFB590B2093"
+              },
+              "expr": "increase(dolphinscheduler_process_instance_failure_count_total{}[1m])",
+              "refId": "A"
+            }
+          ],
+          "title": "Process Instance Failure/1m",
+          "type": "timeseries"
+        }
+      ],
+      "title": "ProcessInstance",
+      "type": "row"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "id": 172,
+      "panels": [],
+      "title": "Task",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "mappings": [],
           "thresholds": {
@@ -71,541 +1551,771 @@
               {
                 "color": "green",
                 "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
               }
             ]
-          },
-          "unit": "JOBS"
+          }
         },
         "overrides": []
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 8,
         "x": 0,
-        "y": 1
+        "y": 4
       },
-      "id": 63,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
+      "id": 178,
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
         },
-        "text": {},
-        "textMode": "auto"
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
-      "pluginVersion": "8.2.3",
       "targets": [
         {
-          "exemplar": true,
-          "expr": "sum(quartz_job_executed_total)",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "",
-          "metric": "",
-          "refId": "A",
-          "step": 14400
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_dispatch_count_total{}[1m]))",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_dispatch_failed_count_total{}[1m]))",
+          "hide": false,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_dispatch_error_count_total{}[1m]))",
+          "hide": false,
+          "refId": "C"
         }
       ],
-      "title": "Job Total Count",
-      "type": "stat"
+      "title": "Task Dispatch Count/1m",
+      "type": "timeseries"
     },
     {
-      "cacheTimeout": null,
-      "datasource": "Prometheus",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "mappings": [],
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "red",
+                "color": "green",
                 "value": null
               },
               {
-                "color": "green",
+                "color": "red",
                 "value": 80
               }
             ]
-          },
-          "unit": "percent"
+          }
         },
         "overrides": []
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 1
+        "w": 8,
+        "x": 8,
+        "y": 4
       },
-      "id": 144,
-      "interval": null,
-      "links": [],
-      "maxDataPoints": 100,
+      "id": 180,
       "options": {
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
         },
-        "showThresholdLabels": false,
-        "showThresholdMarkers": true,
-        "text": {}
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
-      "pluginVersion": "8.2.3",
       "targets": [
         {
-          "exemplar": true,
-          "expr": "sum(quartz_job_executed_total{result=\"success\"}) / sum(quartz_job_executed_total) * 100",
-          "format": "time_series",
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "",
-          "metric": "",
-          "refId": "A",
-          "step": 14400
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_submit_count_total{}[1m]))",
+          "refId": "A"
         }
       ],
-      "title": "Job Successful Rate",
-      "type": "gauge"
+      "title": "Task Submit Count/1m",
+      "type": "timeseries"
     },
     {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "description": "",
-      "fill": 1,
-      "fillGradient": 0,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
       "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 9
-      },
-      "hiddenSeries": false,
-      "id": 139,
-      "legend": {
-        "alignAsTable": false,
-        "avg": false,
-        "current": true,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 4
+      },
+      "id": 182,
       "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.3",
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
       "targets": [
         {
-          "exemplar": true,
-          "expr": "sum(quartz_job_executed_total{})",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "Total",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_finish_count_total{}[1m]))",
           "refId": "A"
-        },
-        {
-          "exemplar": true,
-          "expr": "quartz_job_executed_total{result=\"success\"}",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "Successful",
-          "refId": "B"
-        },
-        {
-          "exemplar": true,
-          "expr": "quartz_job_executed_total{result=\"failure\"}",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "Failed ({{exception}})",
-          "refId": "C"
         }
       ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Job Executed Count",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
+      "title": "Task Finish Count/1m",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
       },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
       },
-      "yaxes": [
-        {
-          "$$hashKey": "object:1516",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 12
+      },
+      "id": 184,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
         },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
         {
-          "$$hashKey": "object:1517",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_success_count_total{}[1m]))",
+          "refId": "A"
         }
       ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+      "title": "Task Success Count/1m",
+      "type": "timeseries"
     },
     {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "description": "",
-      "fill": 1,
-      "fillGradient": 0,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
       "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 9
-      },
-      "hiddenSeries": false,
-      "id": 119,
-      "legend": {
-        "alignAsTable": false,
-        "avg": false,
-        "current": true,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 12
+      },
+      "id": 186,
       "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.3",
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
       "targets": [
         {
-          "exemplar": true,
-          "expr": "sum(increase(quartz_job_executed_total[1m]))",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "Total",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_failure_count_total{}[1m]))",
           "refId": "A"
-        },
-        {
-          "exemplar": true,
-          "expr": "increase(quartz_job_executed_total{result=\"success\"}[1m])",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "Successful",
-          "refId": "B"
-        },
-        {
-          "exemplar": true,
-          "expr": "increase(quartz_job_executed_total{result=\"failure\"}[1m])",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 2,
-          "legendFormat": "Failed ({{exception}})",
-          "refId": "C"
         }
       ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Job Executed Count / Minute",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
+      "title": "Task failure Count/1m",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
       },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 12
       },
-      "yaxes": [
-        {
-          "$$hashKey": "object:1516",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
+      "id": 188,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
         },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
         {
-          "$$hashKey": "object:1517",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_timeout_count_total{}[1m]))",
+          "refId": "A"
         }
       ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+      "title": "Task Timeout Count/1m",
+      "type": "timeseries"
     },
     {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "Prometheus",
-      "fill": 1,
-      "fillGradient": 0,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
       "gridPos": {
-        "h": 9,
-        "w": 12,
+        "h": 8,
+        "w": 8,
         "x": 0,
-        "y": 18
-      },
-      "hiddenSeries": false,
-      "id": 101,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
+        "y": 20
+      },
+      "id": 190,
       "options": {
-        "alertThreshold": true
-      },
-      "percentage": false,
-      "pluginVersion": "8.2.3",
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
       "targets": [
         {
-          "exemplar": true,
-          "expr": "rate(quartz_job_execution_seconds_sum[1m])/rate(quartz_job_execution_seconds_count[1m])",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "avg ({{exception}})",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_retry_count_total{}[1m]))",
           "refId": "A"
-        },
-        {
-          "exemplar": true,
-          "expr": "quartz_job_execution_seconds_max",
-          "format": "time_series",
-          "hide": false,
-          "instant": false,
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "max ({{exception}})",
-          "refId": "B"
         }
       ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "Job Execution Time",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
+      "title": "Task Retry Count/1m",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
       },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
       },
-      "yaxes": [
-        {
-          "$$hashKey": "object:1671",
-          "format": "s",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 20
+      },
+      "id": 192,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
         },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
         {
-          "$$hashKey": "object:1672",
-          "format": "short",
-          "label": "",
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_stop_count_total{}[1m]))",
+          "refId": "A"
         }
       ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+      "title": "Task Stop Count/1m",
+      "type": "timeseries"
     },
     {
-      "cards": {
-        "cardPadding": null,
-        "cardRound": null
-      },
-      "color": {
-        "cardColor": "#F2495C",
-        "colorScale": "sqrt",
-        "colorScheme": "interpolateReds",
-        "exponent": 0.5,
-        "mode": "opacity"
-      },
-      "dataFormat": "timeseries",
-      "datasource": "Prometheus",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
       "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 18
-      },
-      "heatmap": {},
-      "hideZeroBuckets": false,
-      "highlightCards": true,
-      "id": 146,
-      "legend": {
-        "show": true
-      },
-      "pluginVersion": "8.2.3",
-      "reverseYBuckets": false,
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 20
+      },
+      "id": 194,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
       "targets": [
         {
-          "exemplar": true,
-          "expr": "histogram_quantile(0.95, sum(rate(quartz_job_execution_seconds_bucket[5m])) by (le))",
-          "interval": "",
-          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(increase(dolphinscheduler_task_failover_count_total{}[1m]))",
           "refId": "A"
         }
       ],
-      "title": "Job Execution Time Distribution",
-      "tooltip": {
-        "show": true,
-        "showHistogram": false
-      },
-      "type": "heatmap",
-      "xAxis": {
-        "show": true
-      },
-      "xBucketNumber": null,
-      "xBucketSize": null,
-      "yAxis": {
-        "decimals": null,
-        "format": "s",
-        "logBase": 1,
-        "max": null,
-        "min": null,
-        "show": true,
-        "splitFactor": null
-      },
-      "yBucketBound": "auto",
-      "yBucketNumber": null,
-      "yBucketSize": null
+      "title": "Task Failover Count/1m",
+      "type": "timeseries"
     }
   ],
   "refresh": "5s",
-  "schemaVersion": 31,
+  "schemaVersion": 36,
   "style": "dark",
   "tags": [],
   "templating": {
     "list": [
       {
-        "allValue": null,
         "current": {
           "selected": false,
-          "text": "master",
-          "value": "master"
+          "text": "master-server",
+          "value": "master-server"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
         },
-        "datasource": "Prometheus",
         "definition": "",
-        "description": null,
-        "error": null,
         "hide": 0,
         "includeAll": false,
         "label": "Application",
@@ -627,16 +2337,16 @@
       },
       {
         "allFormat": "glob",
-        "allValue": null,
         "current": {
           "selected": false,
           "text": "host.docker.internal:5679",
           "value": "host.docker.internal:5679"
         },
-        "datasource": "Prometheus",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
         "definition": "",
-        "description": null,
-        "error": null,
         "hide": 0,
         "includeAll": false,
         "label": "Instance",
@@ -659,16 +2369,16 @@
       },
       {
         "allFormat": "glob",
-        "allValue": null,
         "current": {
           "selected": false,
           "text": "All",
           "value": "$__all"
         },
-        "datasource": "Prometheus",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
         "definition": "",
-        "description": null,
-        "error": null,
         "hide": 0,
         "includeAll": true,
         "label": "JVM Memory Pools Heap",
@@ -691,16 +2401,16 @@
       },
       {
         "allFormat": "glob",
-        "allValue": null,
         "current": {
           "selected": false,
           "text": "All",
           "value": "$__all"
         },
-        "datasource": "Prometheus",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
         "definition": "",
-        "description": null,
-        "error": null,
         "hide": 0,
         "includeAll": true,
         "label": "JVM Memory Pools Non-Heap",
@@ -724,7 +2434,7 @@
     ]
   },
   "time": {
-    "from": "now-30m",
+    "from": "now-5m",
     "to": "now"
   },
   "timepicker": {
@@ -756,5 +2466,6 @@
   "timezone": "browser",
   "title": "Master",
   "uid": "6XgATOcnz",
-  "version": 20
-}
+  "version": 2,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerWorker.json b/dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerWorker.json
new file mode 100644
index 0000000000..765f40bf8f
--- /dev/null
+++ b/dolphinscheduler-meter/src/main/resources/grafana/DolphinSchedulerWorker.json
@@ -0,0 +1,1598 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 24,
+      "panels": [],
+      "title": "Worker",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 28,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "process_cpu_usage{application=\"worker-server\"}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Worker CPU Usage",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 30,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "/^Time$/",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": false
+      },
+      "pluginVersion": "8.5.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "process_start_time_seconds{}",
+          "refId": "A"
+        }
+      ],
+      "title": "Worker Start Time",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 9
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "increase(dolphinscheduler_worker_submit_queue_is_full_count_total{}[1m])",
+          "refId": "A"
+        }
+      ],
+      "title": "Worker Submit Queue Full/1m",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 9,
+        "x": 8,
+        "y": 9
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "increase(dolphinscheduler_worker_overload_count_total{}[1m])",
+          "refId": "A"
+        }
+      ],
+      "title": "Worker Overload Increase/1m",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 9
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "dolphinscheduler_worker_running_task_gauge{}",
+          "refId": "A"
+        }
+      ],
+      "title": "Worker Running Task",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 17
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "8.5.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(dolphinscheduler_task_execution_count_total)",
+          "refId": "A"
+        }
+      ],
+      "title": "Task Total Count",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 17
+      },
+      "id": 12,
+      "options": {
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "8.5.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "sum(dolphinscheduler_task_execution_count_total{result=\"success\"}) / sum(dolphinscheduler_task_execution_count_total) * 100",
+          "refId": "A"
+        }
+      ],
+      "title": "Task Successful Rate",
+      "type": "gauge"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "millse"
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 25
+      },
+      "hiddenSeries": false,
+      "id": 16,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.3",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "rate(dolphinscheduler_task_execution_timer_seconds_sum[1m])/rate(dolphinscheduler_task_execution_timer_seconds_count[1m])",
+          "legendFormat": "avg ({{exception}})",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "dolphinscheduler_task_execution_timer_seconds_max",
+          "hide": false,
+          "legendFormat": "max ({{exception}})",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Task Execution Time",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:236",
+          "format": "millse",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:237",
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 25
+      },
+      "hiddenSeries": false,
+      "id": 18,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.5.3",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum(increase(dolphinscheduler_task_execution_count_total[1m]))",
+          "legendFormat": "Total",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "increase(dolphinscheduler_task_execution_count_total{result=\"success\"}[1m])",
+          "hide": false,
+          "legendFormat": "Successful",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "Task Execution Total/Success",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 33
+      },
+      "id": 26,
+      "panels": [],
+      "title": "Task",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 34
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"SHELL\"}[1d]))",
+          "legendFormat": "SHELL",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Shell Task Execute/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 9,
+        "x": 8,
+        "y": 34
+      },
+      "id": 20,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"unknown\"}[1d]))",
+          "legendFormat": "unknown",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Unkown Task Execution Total/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 34
+      },
+      "id": 22,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"CONDITIONS\"}[1d]))",
+          "legendFormat": "CONDITIONS",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Condition Task Execution Total/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 42
+      },
+      "id": 32,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"BLOCKING\"}[1d]))",
+          "legendFormat": "BLOCKING",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Blocking Task Execute/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 9,
+        "x": 8,
+        "y": 42
+      },
+      "id": 34,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"DATAX\"}[1d]))",
+          "legendFormat": "DATAX",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "DATAX Task Execute/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 42
+      },
+      "id": 36,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"DATA_QUALITY\"}[1d]))",
+          "legendFormat": "DATA_QUALITY",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "DATA_QUALITY Task Execute/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 50
+      },
+      "id": 38,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"DATA_QUALITY\"}[1d]))",
+          "legendFormat": "DATA_QUALITY",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "DATA_QUALITY Task Execute/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 9,
+        "x": 8,
+        "y": 50
+      },
+      "id": 40,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"DEPENDENT\"}[1d]))",
+          "legendFormat": "DEPENDENT",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "DEPENDENT Task Execute/1d",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 7,
+        "x": 17,
+        "y": 50
+      },
+      "id": 42,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(dolphinscheduler_task_execute_count_total{task_type=\"EMR\"}[1d]))",
+          "legendFormat": "EMR",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "EMR Task Execute/1d",
+      "type": "timeseries"
+    }
+  ],
+  "schemaVersion": 36,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Worker",
+  "uid": "6wXtd3r7k",
+  "version": 2,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java
index 82ae175e29..af51831068 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/StateEventCallbackService.java
@@ -86,7 +86,7 @@ public class StateEventCallbackService {
         return null;
     }
 
-    public int pause(int ntries) {
+    public long pause(int ntries) {
         return SLEEP_TIME_MILLIS * RETRY_BACKOFF[ntries % RETRY_BACKOFF.length];
     }
 
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index eba7ced00a..a976c05c99 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.service.process;
 
+import io.micrometer.core.annotation.Counted;
 import static java.util.stream.Collectors.toSet;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
@@ -391,6 +392,7 @@ public class ProcessServiceImpl implements ProcessService {
      * @return create result
      */
     @Override
+    @Counted("dolphinscheduler_create_command_count")
     public int createCommand(Command command) {
         int result = 0;
         if (command != null) {
diff --git a/dolphinscheduler-standalone-server/src/main/resources/banner.txt b/dolphinscheduler-standalone-server/src/main/resources/banner.txt
index 0b1f0023a8..4040c5c00a 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/banner.txt
+++ b/dolphinscheduler-standalone-server/src/main/resources/banner.txt
@@ -8,5 +8,5 @@ ${AnsiColor.BLUE}${AnsiStyle.BOLD}
               |_|
 ================================================================================
 ${AnsiColor.BLUE}${AnsiStyle.BOLD}
-::  Dolphinscheduler standalone server ::  ${application.formatted-version}
+::  DolphinScheduler standalone server ::  ${application.formatted-version}
 ${AnsiStyle.NORMAL}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java
new file mode 100644
index 0000000000..c45275a8a3
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/TaskMetrics.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.metrics;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.ServiceLoader;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+
+public final class TaskMetrics {
+
+    private TaskMetrics() {
+        throw new UnsupportedOperationException("Utility class");
+    }
+
+    private static Map<String, Counter> TASK_TYPE_EXECUTE_COUNTER = new HashMap<>();
+    private static final Counter UNKNOWN_TASK_EXECUTE_COUNTER =
+            Counter.builder("dolphinscheduler_task_execute_count")
+                    .tag("task_type", "unknown")
+                    .description("task execute counter")
+                    .register(Metrics.globalRegistry);
+
+    static {
+        for (TaskChannelFactory taskChannelFactory : ServiceLoader.load(TaskChannelFactory.class)) {
+            TASK_TYPE_EXECUTE_COUNTER.put(
+                    taskChannelFactory.getName(),
+                    Counter.builder("dolphinscheduler_task_execute_count")
+                            .tag("task_type", taskChannelFactory.getName())
+                            .description("task execute counter")
+                            .register(Metrics.globalRegistry)
+            );
+        }
+    }
+
+    public static void incrTaskTypeExecuteCount(String taskType) {
+        TASK_TYPE_EXECUTE_COUNTER.getOrDefault(taskType, UNKNOWN_TASK_EXECUTE_COUNTER).increment();
+    }
+
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java
new file mode 100644
index 0000000000..9613211f43
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.metrics;
+
+import java.util.function.Supplier;
+
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.Metrics;
+
+public final class WorkerServerMetrics {
+
+    public WorkerServerMetrics() {
+        throw new UnsupportedOperationException("Utility class");
+    }
+
+    private static final Counter WORKER_OVERLOAD_COUNTER =
+            Counter.builder("dolphinscheduler_worker_overload_count")
+                    .description("worker load count")
+                    .register(Metrics.globalRegistry);
+
+    private static final Counter WORKER_SUBMIT_QUEUE_IS_FULL_COUNTER =
+            Counter.builder("dolphinscheduler_worker_submit_queue_is_full_count")
+                    .description("worker task submit queue is full count")
+                    .register(Metrics.globalRegistry);
+
+    public static void incWorkerOverloadCount() {
+        WORKER_OVERLOAD_COUNTER.increment();
+    }
+
+    public static void incWorkerSubmitQueueIsFullCount() {
+        WORKER_SUBMIT_QUEUE_IS_FULL_COUNTER.increment();
+    }
+
+    public static void registerWorkerRunningTaskGauge(Supplier<Number> supplier) {
+        Gauge.builder("dolphinscheduler_worker_running_task_gauge", supplier)
+                .description("worker running task gauge")
+                .register(Metrics.globalRegistry);
+
+    }
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
index 2a92222b0a..5fff6a24df 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
@@ -118,7 +118,7 @@ public class TaskCallbackService {
         return null;
     }
 
-    public int pause(int ntries) {
+    public long pause(int ntries) {
         return SLEEP_TIME_MILLIS * RETRY_BACKOFF[ntries % RETRY_BACKOFF.length];
     }
 
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
index a376587aad..34382257d7 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
@@ -34,11 +34,14 @@ import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel;
 import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
 import org.apache.dolphinscheduler.server.utils.LogUtils;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.metrics.TaskMetrics;
 import org.apache.dolphinscheduler.server.worker.runner.TaskExecuteThread;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
 import org.apache.dolphinscheduler.service.task.TaskPluginManager;
 
+import org.apache.commons.lang.SystemUtils;
+
 import java.util.Date;
 
 import org.slf4j.Logger;
@@ -48,6 +51,8 @@ import org.springframework.stereotype.Component;
 
 import com.google.common.base.Preconditions;
 
+import io.micrometer.core.annotation.Counted;
+import io.micrometer.core.annotation.Timed;
 import io.netty.channel.Channel;
 
 /**
@@ -85,6 +90,8 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
     @Autowired
     private WorkerManagerThread workerManager;
 
+    @Counted(value = "dolphinscheduler_task_execution_count", description = "task execute total count")
+    @Timed(value = "dolphinscheduler_task_execution_timer", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
     @Override
     public void process(Channel channel, Command command) {
         Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(),
@@ -106,6 +113,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
             logger.error("task execution context is null");
             return;
         }
+        TaskMetrics.incrTaskTypeExecuteCount(taskExecutionContext.getTaskType());
 
         // set cache, it will be used when kill task
         TaskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java
index ff050370cc..67abf523ed 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerExecService.java
@@ -17,6 +17,8 @@
 
 package org.apache.dolphinscheduler.server.worker.runner;
 
+import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics;
+
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -52,6 +54,7 @@ public class WorkerExecService {
         this.execService = execService;
         this.listeningExecutorService = MoreExecutors.listeningDecorator(this.execService);
         this.taskExecuteThreadMap = taskExecuteThreadMap;
+        WorkerServerMetrics.registerWorkerRunningTaskGauge(taskExecuteThreadMap::size);
     }
 
     public void submit(TaskExecuteThread taskExecuteThread) {
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
index 60f752401d..2702d4af42 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics;
 import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
 
 import java.util.concurrent.ConcurrentHashMap;
diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml
index dd353f0495..397e533db9 100644
--- a/dolphinscheduler-worker/src/main/resources/application.yaml
+++ b/dolphinscheduler-worker/src/main/resources/application.yaml
@@ -99,3 +99,6 @@ spring:
       on-profile: mysql
   datasource:
     driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
+    username: root
+    password: root
diff --git a/dolphinscheduler-worker/src/main/resources/banner.txt b/dolphinscheduler-worker/src/main/resources/banner.txt
index 7f93ee991b..bf195ee524 100644
--- a/dolphinscheduler-worker/src/main/resources/banner.txt
+++ b/dolphinscheduler-worker/src/main/resources/banner.txt
@@ -8,5 +8,5 @@ ${AnsiColor.BLUE}${AnsiStyle.BOLD}
               |_|
 ================================================================================
 ${AnsiColor.BLUE}${AnsiStyle.BOLD}
-::  Dolphinscheduler work server ::  ${application.formatted-version}
+::  DolphinScheduler work server ::  ${application.formatted-version}
 ${AnsiStyle.NORMAL}


[dolphinscheduler] 13/29: [Bug] [Master] Worker failover will cause task cannot be failover (#10631)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit db31deb54f2824b6fafe68f86713e89fa671f8e4
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jun 28 16:08:35 2022 +0800

    [Bug] [Master] Worker failover will cause task cannot be failover (#10631)
    
    * fix worker failover may lose event
    
    (cherry picked from commit 66624c5c86dd0bb2ae658eaf68691e03c86ae5e0)
---
 .../dolphinscheduler/common/enums/StateEvent.java  |  77 +----
 .../dolphinscheduler/common/utils/OSUtils.java     |  21 +-
 .../cache/ProcessInstanceExecCacheManager.java     |   4 +-
 .../impl/ProcessInstanceExecCacheManagerImpl.java  |   7 +-
 .../master/consumer/TaskPriorityQueueConsumer.java |   8 +-
 .../master/metrics/ProcessInstanceMetrics.java     |  51 ++-
 .../master/runner/MasterSchedulerService.java      |  73 ++--
 .../master/runner/StateWheelExecuteThread.java     |  14 +-
 .../master/runner/WorkflowExecuteRunnable.java     |  27 +-
 .../master/runner/WorkflowExecuteThreadPool.java   |  12 +-
 .../master/runner/task/BaseTaskProcessor.java      |  21 +-
 .../server/master/runner/task/TaskInstanceKey.java |  62 +---
 .../server/master/service/FailoverService.java     | 104 +++---
 .../src/main/resources/application.yaml            |   2 +-
 .../ProcessInstanceExecCacheManagerImplTest.java   |   2 +-
 .../master/dispatch/ExecutionContextTestUtils.java |   3 +-
 .../executor/NettyExecutorManagerTest.java         |   2 +-
 .../server/master/service/FailoverServiceTest.java |  21 +-
 .../dolphinscheduler/meter/MeterConfiguration.java |   4 -
 .../dolphinscheduler/remote/command/Command.java   |   3 +
 .../remote/command/TaskExecuteRequestCommand.java  |  42 +--
 .../server/utils/ProcessUtils.java                 |   7 +-
 .../service/process/ProcessService.java            |   8 +-
 .../service/process/ProcessServiceImpl.java        |  23 +-
 .../service/process/ProcessServiceTest.java        |  52 +--
 .../plugin/task/api/TaskExecutionContext.java      | 383 +--------------------
 .../plugin/task/api/enums/ExecutionStatus.java     |  23 ++
 .../server/worker/metrics/WorkerServerMetrics.java |  27 +-
 .../worker/processor/TaskCallbackService.java      |  26 +-
 .../worker/processor/TaskExecuteProcessor.java     |   3 +-
 .../worker/runner/RetryReportTaskStatusThread.java |  68 ++--
 .../server/worker/runner/TaskExecuteThread.java    |   3 +-
 .../worker/processor/TaskExecuteProcessorTest.java |   4 +-
 33 files changed, 386 insertions(+), 801 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
index 54c7835f0b..7f4be924dd 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StateEvent.java
@@ -20,10 +20,12 @@ package org.apache.dolphinscheduler.common.enums;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 
 import io.netty.channel.Channel;
+import lombok.Data;
 
 /**
  * state event
  */
+@Data
 public class StateEvent {
 
     /**
@@ -45,79 +47,4 @@ public class StateEvent {
 
     private Channel channel;
 
-    public ExecutionStatus getExecutionStatus() {
-        return executionStatus;
-    }
-
-    public void setExecutionStatus(ExecutionStatus executionStatus) {
-        this.executionStatus = executionStatus;
-    }
-
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public long getTaskCode() {
-        return taskCode;
-    }
-
-    public int getProcessInstanceId() {
-        return processInstanceId;
-    }
-
-    public void setProcessInstanceId(int processInstanceId) {
-        this.processInstanceId = processInstanceId;
-    }
-
-    public String getContext() {
-        return context;
-    }
-
-    public void setContext(String context) {
-        this.context = context;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-    }
-
-    public void setTaskCode(long taskCode) {
-        this.taskCode = taskCode;
-    }
-
-    public Channel getChannel() {
-        return channel;
-    }
-
-    public void setChannel(Channel channel) {
-        this.channel = channel;
-    }
-
-    @Override
-    public String toString() {
-        return "State Event :"
-                + "key: " + key
-                + " type: " + type.toString()
-                + " executeStatus: " + executionStatus
-                + " task instance id: " + taskInstanceId
-                + " process instance id: " + processInstanceId
-                + " context: " + context
-                ;
-    }
-
-    public String getKey() {
-        return key;
-    }
-
-    public void setKey(String key) {
-        this.key = key;
-    }
-
-    public void setType(StateEventType type) {
-        this.type = type;
-    }
-
-    public StateEventType getType() {
-        return this.type;
-    }
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
index a4e79236ca..813e84630b 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
@@ -423,24 +423,27 @@ public class OSUtils {
     }
 
     /**
-     * check memory and cpu usage
+     * Check memory and cpu usage is overload the given thredshod.
      *
-     * @param maxCpuloadAvg maxCpuloadAvg
+     * @param maxCpuLoadAvg  maxCpuLoadAvg
      * @param reservedMemory reservedMemory
-     * @return check memory and cpu usage
+     * @return True, if the cpu or memory exceed the given thredshod.
      */
-    public static Boolean checkResource(double maxCpuloadAvg, double reservedMemory) {
+    public static Boolean isOverload(double maxCpuLoadAvg, double reservedMemory) {
         // system load average
         double loadAverage = loadAverage();
         // system available physical memory
         double availablePhysicalMemorySize = availablePhysicalMemorySize();
-        if (loadAverage > maxCpuloadAvg || availablePhysicalMemorySize < reservedMemory) {
-            logger.warn("current cpu load average {} is too high or available memory {}G is too low, under max.cpuload.avg={} and reserved.memory={}G",
-                    loadAverage, availablePhysicalMemorySize, maxCpuloadAvg, reservedMemory);
-            return false;
-        } else {
+        if (loadAverage > maxCpuLoadAvg || availablePhysicalMemorySize < reservedMemory) {
+            logger.warn(
+                "Current cpu load average {} is too high or available memory {}G is too low, under max.cpuLoad.avg={} and reserved.memory={}G",
+                loadAverage,
+                availablePhysicalMemorySize,
+                maxCpuLoadAvg,
+                reservedMemory);
             return true;
         }
+        return false;
     }
 
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
index 2f5f6dc472..59e064105a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/ProcessInstanceExecCacheManager.java
@@ -17,6 +17,8 @@
 
 package org.apache.dolphinscheduler.server.master.cache;
 
+import lombok.NonNull;
+
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
 import java.util.Collection;
@@ -55,7 +57,7 @@ public interface ProcessInstanceExecCacheManager {
      * @param processInstanceId     processInstanceId
      * @param workflowExecuteThread if it is null, will not be cached
      */
-    void cache(int processInstanceId, WorkflowExecuteRunnable workflowExecuteThread);
+    void cache(int processInstanceId, @NonNull WorkflowExecuteRunnable workflowExecuteThread);
 
     /**
      * get all WorkflowExecuteThread from cache
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
index dc562d37bd..8f00029a3e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImpl.java
@@ -30,6 +30,8 @@ import org.springframework.stereotype.Component;
 
 import com.google.common.collect.ImmutableList;
 
+import lombok.NonNull;
+
 /**
  * cache of process instance id and WorkflowExecuteThread
  */
@@ -59,10 +61,7 @@ public class ProcessInstanceExecCacheManagerImpl implements ProcessInstanceExecC
     }
 
     @Override
-    public void cache(int processInstanceId, WorkflowExecuteRunnable workflowExecuteThread) {
-        if (workflowExecuteThread == null) {
-            return;
-        }
+    public void cache(int processInstanceId, @NonNull WorkflowExecuteRunnable workflowExecuteThread) {
         processInstanceExecMaps.put(processInstanceId, workflowExecuteThread);
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index eab33a07a6..e8cf4b2919 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -131,9 +131,8 @@ public class TaskPriorityQueueConsumer extends BaseDaemonThread {
                     for (TaskPriority dispatchFailedTask : failedDispatchTasks) {
                         taskPriorityQueue.put(dispatchFailedTask);
                     }
-                    // If there are tasks in a cycle that cannot find the worker group,
-                    // sleep for 1 second
-                    if (taskPriorityQueue.size() <= failedDispatchTasks.size()) {
+                    // If the all task dispatch failed, will sleep for 1s to avoid the master cpu higher.
+                    if (fetchTaskNum == failedDispatchTasks.size()) {
                         TimeUnit.MILLISECONDS.sleep(Constants.SLEEP_TIME_MILLIS);
                     }
                 }
@@ -218,8 +217,7 @@ public class TaskPriorityQueueConsumer extends BaseDaemonThread {
     }
 
     private Command toCommand(TaskExecutionContext taskExecutionContext) {
-        TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand();
-        requestCommand.setTaskExecutionContext(JSONUtils.toJsonString(taskExecutionContext));
+        TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(taskExecutionContext);
         return requestCommand.convert2Command();
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
index 9e9b11c95d..957a3ecb7b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/ProcessInstanceMetrics.java
@@ -17,11 +17,13 @@
 
 package org.apache.dolphinscheduler.server.master.metrics;
 
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Gauge;
 import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Timer;
 
 public final class ProcessInstanceMetrics {
 
@@ -29,15 +31,25 @@ public final class ProcessInstanceMetrics {
         throw new UnsupportedOperationException("Utility class");
     }
 
+    private static final Timer COMMAND_QUERY_TIMETER =
+        Timer.builder("ds.workflow.command.query.duration")
+            .description("Command query duration")
+            .register(Metrics.globalRegistry);
+
+    private static final Timer PROCESS_INSTANCE_GENERATE_TIMER =
+        Timer.builder("ds.workflow.instance.generate.duration")
+            .description("Process instance generated duration")
+            .register(Metrics.globalRegistry);
+
     private static final Counter PROCESS_INSTANCE_SUBMIT_COUNTER =
-            Counter.builder("dolphinscheduler_process_instance_submit_count")
-                    .description("Process instance submit total count")
-                    .register(Metrics.globalRegistry);
+        Counter.builder("ds.workflow.instance.submit.count")
+            .description("Process instance submit total count")
+            .register(Metrics.globalRegistry);
 
     private static final Counter PROCESS_INSTANCE_TIMEOUT_COUNTER =
-            Counter.builder("dolphinscheduler_process_instance_timeout_count")
-                    .description("Process instance timeout total count")
-                    .register(Metrics.globalRegistry);
+        Counter.builder("ds.workflow.instance.timeout.count")
+            .description("Process instance timeout total count")
+            .register(Metrics.globalRegistry);
 
     private static final Counter PROCESS_INSTANCE_FINISH_COUNTER =
             Counter.builder("dolphinscheduler_process_instance_finish_count")
@@ -55,19 +67,27 @@ public final class ProcessInstanceMetrics {
                     .register(Metrics.globalRegistry);
 
     private static final Counter PROCESS_INSTANCE_STOP_COUNTER =
-            Counter.builder("dolphinscheduler_process_instance_stop_count")
-                    .description("Process instance stop total count")
-                    .register(Metrics.globalRegistry);
+        Counter.builder("ds.workflow.instance.stop.count")
+            .description("Process instance stop total count")
+            .register(Metrics.globalRegistry);
 
     private static final Counter PROCESS_INSTANCE_FAILOVER_COUNTER =
-            Counter.builder("dolphinscheduler_process_instance_failover_count")
-                    .description("Process instance failover total count")
-                    .register(Metrics.globalRegistry);
+        Counter.builder("ds.workflow.instance.failover.count")
+            .description("Process instance failover total count")
+            .register(Metrics.globalRegistry);
+
+    public static void recordCommandQueryTime(long milliseconds) {
+        COMMAND_QUERY_TIMETER.record(milliseconds, TimeUnit.MILLISECONDS);
+    }
+
+    public static void recordProcessInstanceGenerateTime(long milliseconds) {
+        PROCESS_INSTANCE_GENERATE_TIMER.record(milliseconds, TimeUnit.MILLISECONDS);
+    }
 
     public static synchronized void registerProcessInstanceRunningGauge(Supplier<Number> function) {
-        Gauge.builder("dolphinscheduler_process_instance_running_gauge", function)
-                .description("The current running process instance count")
-                .register(Metrics.globalRegistry);
+        Gauge.builder("ds.workflow.instance.running", function)
+            .description("The current running process instance count")
+            .register(Metrics.globalRegistry);
     }
 
     public static void incProcessInstanceSubmit() {
@@ -97,5 +117,4 @@ public final class ProcessInstanceMetrics {
     public static void incProcessInstanceFailover() {
         PROCESS_INSTANCE_FAILOVER_COUNTER.increment();
     }
-
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index 023d871355..6de3541dae 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheM
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutorManager;
 import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
+import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
 import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
 import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -61,31 +62,19 @@ public class MasterSchedulerService extends BaseDaemonThread {
      */
     private static final Logger logger = LoggerFactory.getLogger(MasterSchedulerService.class);
 
-    /**
-     * dolphinscheduler database interface
-     */
     @Autowired
     private ProcessService processService;
 
-    /**
-     * master config
-     */
     @Autowired
     private MasterConfig masterConfig;
 
-    /**
-     * alert manager
-     */
     @Autowired
     private ProcessAlertManager processAlertManager;
 
-    /**
-     * netty remoting client
-     */
     private NettyRemotingClient nettyRemotingClient;
 
     @Autowired
-    NettyExecutorManager nettyExecutorManager;
+    private NettyExecutorManager nettyExecutorManager;
 
     /**
      * master prepare exec service
@@ -104,6 +93,8 @@ public class MasterSchedulerService extends BaseDaemonThread {
     @Autowired
     private StateWheelExecuteThread stateWheelExecuteThread;
 
+    private String masterAddress;
+
     protected MasterSchedulerService() {
         super("MasterCommandLoopThread");
     }
@@ -115,6 +106,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
         this.masterPrepareExecService = (ThreadPoolExecutor) ThreadUtils.newDaemonFixedThreadExecutor("MasterPreExecThread", masterConfig.getPreExecThreads());
         NettyClientConfig clientConfig = new NettyClientConfig();
         this.nettyRemotingClient = new NettyRemotingClient(clientConfig);
+        this.masterAddress = NetUtils.getAddr(masterConfig.getListenPort());
     }
 
     @Override
@@ -138,13 +130,13 @@ public class MasterSchedulerService extends BaseDaemonThread {
     public void run() {
         while (Stopper.isRunning()) {
             try {
-                boolean runCheckFlag = OSUtils.checkResource(masterConfig.getMaxCpuLoadAvg(), masterConfig.getReservedMemory());
-                if (!runCheckFlag) {
+                boolean isOverload = OSUtils.isOverload(masterConfig.getMaxCpuLoadAvg(), masterConfig.getReservedMemory());
+                if (isOverload) {
                     MasterServerMetrics.incMasterOverload();
                     Thread.sleep(Constants.SLEEP_TIME_MILLIS);
                     continue;
                 }
-                scheduleProcess();
+                scheduleWorkflow();
             } catch (InterruptedException interruptedException) {
                 logger.warn("Master schedule service interrupted, close the loop", interruptedException);
                 Thread.currentThread().interrupt();
@@ -156,13 +148,12 @@ public class MasterSchedulerService extends BaseDaemonThread {
     }
 
     /**
-     * 1. get command by slot
-     * 2. donot handle command if slot is empty
+     * Query command from database by slot, and transform to workflow instance, then submit to workflowExecuteThreadPool.
      */
-    private void scheduleProcess() throws InterruptedException {
+    private void scheduleWorkflow() throws InterruptedException {
         List<Command> commands = findCommands();
         if (CollectionUtils.isEmpty(commands)) {
-            //indicate that no command ,sleep for 1s
+            // indicate that no command ,sleep for 1s
             Thread.sleep(Constants.SLEEP_TIME_MILLIS);
             return;
         }
@@ -177,7 +168,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
             try {
                 LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
                 logger.info("Master schedule service starting workflow instance");
-                WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
+                final WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
                     processInstance
                     , processService
                     , nettyExecutorManager
@@ -189,9 +180,14 @@ public class MasterSchedulerService extends BaseDaemonThread {
                 if (processInstance.getTimeout() > 0) {
                     stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
                 }
-                workflowExecuteThreadPool.startWorkflow(workflowExecuteRunnable);
+                ProcessInstanceMetrics.incProcessInstanceSubmit();
+                workflowExecuteThreadPool.submit(workflowExecuteRunnable);
                 logger.info("Master schedule service started workflow instance");
 
+            } catch (Exception ex) {
+                processInstanceExecCacheManager.removeByProcessInstanceId(processInstance.getId());
+                stateWheelExecuteThread.removeProcess4TimeoutCheck(processInstance.getId());
+                logger.info("Master submit workflow to thread pool failed, will remove workflow runnable from cache manager", ex);
             } finally {
                 LoggerUtils.removeWorkflowInstanceIdMDC();
             }
@@ -199,21 +195,21 @@ public class MasterSchedulerService extends BaseDaemonThread {
     }
 
     private List<ProcessInstance> command2ProcessInstance(List<Command> commands) throws InterruptedException {
+        long commandTransformStartTime = System.currentTimeMillis();
         logger.info("Master schedule service transforming command to ProcessInstance, commandSize: {}", commands.size());
         List<ProcessInstance> processInstances = Collections.synchronizedList(new ArrayList<>(commands.size()));
         CountDownLatch latch = new CountDownLatch(commands.size());
         for (final Command command : commands) {
             masterPrepareExecService.execute(() -> {
                 try {
+                    // todo: this check is not safe, the slot may change after command transform.
                     // slot check again
                     SlotCheckState slotCheckState = slotCheck(command);
                     if (slotCheckState.equals(SlotCheckState.CHANGE) || slotCheckState.equals(SlotCheckState.INJECT)) {
                         logger.info("Master handle command {} skip, slot check state: {}", command.getId(), slotCheckState);
                         return;
                     }
-                    ProcessInstance processInstance = processService.handleCommand(logger,
-                            getLocalAddress(),
-                            command);
+                    ProcessInstance processInstance = processService.handleCommand(masterAddress, command);
                     if (processInstance != null) {
                         processInstances.add(processInstance);
                         logger.info("Master handle command {} end, create process instance {}", command.getId(), processInstance.getId());
@@ -231,24 +227,26 @@ public class MasterSchedulerService extends BaseDaemonThread {
         latch.await();
         logger.info("Master schedule service transformed command to ProcessInstance, commandSize: {}, processInstanceSize: {}",
             commands.size(), processInstances.size());
+        ProcessInstanceMetrics.recordProcessInstanceGenerateTime(System.currentTimeMillis() - commandTransformStartTime);
         return processInstances;
     }
 
     private List<Command> findCommands() {
+        long scheduleStartTime = System.currentTimeMillis();
+        int thisMasterSlot = ServerNodeManager.getSlot();
+        int masterCount = ServerNodeManager.getMasterSize();
+        if (masterCount <= 0) {
+            logger.warn("Master count: {} is invalid, the current slot: {}", masterCount, thisMasterSlot);
+            return Collections.emptyList();
+        }
         int pageNumber = 0;
         int pageSize = masterConfig.getFetchCommandNum();
-        List<Command> result = new ArrayList<>();
-        if (Stopper.isRunning()) {
-            int thisMasterSlot = ServerNodeManager.getSlot();
-            int masterCount = ServerNodeManager.getMasterSize();
-            if (masterCount > 0) {
-                result = processService.findCommandPageBySlot(pageSize, pageNumber, masterCount, thisMasterSlot);
-                if (CollectionUtils.isNotEmpty(result)) {
-                    logger.info("Master schedule service loop command success, command size: {}, current slot: {}, total slot size: {}",
-                        result.size(), thisMasterSlot, masterCount);
-                }
-            }
+        final List<Command> result = processService.findCommandPageBySlot(pageSize, pageNumber, masterCount, thisMasterSlot);
+        if (CollectionUtils.isNotEmpty(result)) {
+            logger.info("Master schedule service loop command success, command size: {}, current slot: {}, total slot size: {}",
+                result.size(), thisMasterSlot, masterCount);
         }
+        ProcessInstanceMetrics.recordCommandQueryTime(System.currentTimeMillis() - scheduleStartTime);
         return result;
     }
 
@@ -266,7 +264,4 @@ public class MasterSchedulerService extends BaseDaemonThread {
         return state;
     }
 
-    private String getLocalAddress() {
-        return NetUtils.getAddr(masterConfig.getListenPort());
-    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 53acb4ce83..65c7db924d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -59,24 +59,24 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
     private static final Logger logger = LoggerFactory.getLogger(StateWheelExecuteThread.class);
 
     /**
-     * process timeout check list
+     * ProcessInstance timeout check list, element is the processInstanceId.
      */
-    private ConcurrentLinkedQueue<Integer> processInstanceTimeoutCheckList = new ConcurrentLinkedQueue<>();
+    private final ConcurrentLinkedQueue<Integer> processInstanceTimeoutCheckList = new ConcurrentLinkedQueue<>();
 
     /**
      * task time out check list
      */
-    private ConcurrentLinkedQueue<TaskInstanceKey> taskInstanceTimeoutCheckList = new ConcurrentLinkedQueue<>();
+    private final ConcurrentLinkedQueue<TaskInstanceKey> taskInstanceTimeoutCheckList = new ConcurrentLinkedQueue<>();
 
     /**
      * task retry check list
      */
-    private ConcurrentLinkedQueue<TaskInstanceKey> taskInstanceRetryCheckList = new ConcurrentLinkedQueue<>();
+    private final ConcurrentLinkedQueue<TaskInstanceKey> taskInstanceRetryCheckList = new ConcurrentLinkedQueue<>();
 
     /**
      * task state check list
      */
-    private ConcurrentLinkedQueue<TaskInstanceKey> taskInstanceStateCheckList = new ConcurrentLinkedQueue<>();
+    private final ConcurrentLinkedQueue<TaskInstanceKey> taskInstanceStateCheckList = new ConcurrentLinkedQueue<>();
 
     @Autowired
     private MasterConfig masterConfig;
@@ -116,8 +116,8 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
         logger.info("Success add workflow instance into timeout check list");
     }
 
-    public void removeProcess4TimeoutCheck(ProcessInstance processInstance) {
-        boolean removeFlag = processInstanceTimeoutCheckList.remove(processInstance.getId());
+    public void removeProcess4TimeoutCheck(int processInstanceId) {
+        boolean removeFlag = processInstanceTimeoutCheckList.remove(processInstanceId);
         if (removeFlag) {
             logger.info("Success remove workflow instance from timeout check list");
         } else {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index ab74f6e227..7676367a18 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -426,6 +426,7 @@ public class WorkflowExecuteRunnable implements Runnable {
 
         if (task.getState().typeIsFinished()) {
             if (completeTaskMap.containsKey(task.getTaskCode()) && completeTaskMap.get(task.getTaskCode()) == task.getId()) {
+                logger.warn("The task instance is already complete, stateEvent: {}", stateEvent);
                 return true;
             }
             taskFinished(task);
@@ -452,11 +453,9 @@ public class WorkflowExecuteRunnable implements Runnable {
     }
 
     private void taskFinished(TaskInstance taskInstance) {
-        logger.info("work flow {} task id:{} code:{} state:{} ",
-                processInstance.getId(),
-                taskInstance.getId(),
-                taskInstance.getTaskCode(),
-                taskInstance.getState());
+        logger.info("TaskInstance finished task code:{} state:{} ",
+            taskInstance.getTaskCode(),
+            taskInstance.getState());
 
         activeTaskProcessorMaps.remove(taskInstance.getTaskCode());
         stateWheelExecuteThread.removeTask4TimeoutCheck(processInstance, taskInstance);
@@ -472,12 +471,13 @@ public class WorkflowExecuteRunnable implements Runnable {
             }
         } else if (taskInstance.taskCanRetry() && processInstance.getState() != ExecutionStatus.READY_STOP) {
             // retry task
+            logger.info("Retry taskInstance taskInstance state: {}", taskInstance.getState());
             retryTaskInstance(taskInstance);
         } else if (taskInstance.getState().typeIsFailure()) {
             completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
             // There are child nodes and the failure policy is: CONTINUE
-            if (DagHelper.haveAllNodeAfterNode(Long.toString(taskInstance.getTaskCode()), dag)
-                && processInstance.getFailureStrategy() == FailureStrategy.CONTINUE) {
+            if (processInstance.getFailureStrategy() == FailureStrategy.CONTINUE
+                && DagHelper.haveAllNodeAfterNode(Long.toString(taskInstance.getTaskCode()), dag)) {
                 submitPostNode(Long.toString(taskInstance.getTaskCode()));
             } else {
                 errorTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
@@ -486,6 +486,7 @@ public class WorkflowExecuteRunnable implements Runnable {
                 }
             }
         } else if (taskInstance.getState().typeIsFinished()) {
+            // todo: when the task instance type is pause, then it should not in completeTaskMap
             completeTaskMap.put(taskInstance.getTaskCode(), taskInstance.getId());
         }
 
@@ -528,7 +529,7 @@ public class WorkflowExecuteRunnable implements Runnable {
         }
         TaskInstance newTaskInstance = cloneRetryTaskInstance(taskInstance);
         if (newTaskInstance == null) {
-            logger.error("retry fail, new taskInstancce is null, task code:{}, task id:{}", taskInstance.getTaskCode(), taskInstance.getId());
+            logger.error("retry fail, new taskInstance is null, task code:{}, task id:{}", taskInstance.getTaskCode(), taskInstance.getId());
             return;
         }
         waitToRetryTaskInstanceMap.put(newTaskInstance.getTaskCode(), newTaskInstance);
@@ -625,7 +626,7 @@ public class WorkflowExecuteRunnable implements Runnable {
      * check if task instance exist by task code
      */
     public boolean checkTaskInstanceByCode(long taskCode) {
-        if (taskInstanceMap == null || taskInstanceMap.size() == 0) {
+        if (taskInstanceMap.isEmpty()) {
             return false;
         }
         for (TaskInstance taskInstance : taskInstanceMap.values()) {
@@ -640,7 +641,7 @@ public class WorkflowExecuteRunnable implements Runnable {
      * check if task instance exist by id
      */
     public boolean checkTaskInstanceById(int taskInstanceId) {
-        if (taskInstanceMap == null || taskInstanceMap.size() == 0) {
+        if (taskInstanceMap.isEmpty()) {
             return false;
         }
         return taskInstanceMap.containsKey(taskInstanceId);
@@ -688,7 +689,7 @@ public class WorkflowExecuteRunnable implements Runnable {
 
             if (stateEvent.getExecutionStatus() == ExecutionStatus.STOP) {
                 // serial wait execution type needs to wake up the waiting process
-                if (processDefinition.getExecutionType().typeIsSerialWait()){
+                if (processDefinition.getExecutionType().typeIsSerialWait() || processDefinition.getExecutionType().typeIsSerialPriority()) {
                     endProcess();
                     return true;
                 }
@@ -1277,6 +1278,10 @@ public class WorkflowExecuteRunnable implements Runnable {
         }
     }
 
+    public Collection<TaskInstance> getAllTaskInstances() {
+        return taskInstanceMap.values();
+    }
+
     private void setVarPoolValue(Map<String, Property> allProperty, Map<String, TaskInstance> allTaskInstance, TaskInstance preTaskInstance, Property thisProperty) {
         //for this taskInstance all the param in this part is IN.
         thisProperty.setDirect(Direct.IN);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index 031f8986ea..ba097109be 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -52,7 +52,7 @@ import com.google.common.base.Strings;
 import lombok.NonNull;
 
 /**
- * Used to execute {@link WorkflowExecuteRunnable}, when
+ * Used to execute {@link WorkflowExecuteRunnable}.
  */
 @Component
 public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
@@ -100,14 +100,6 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
         logger.info("Submit state event success, stateEvent: {}", stateEvent);
     }
 
-    /**
-     * Start the given workflow.
-     */
-    public void startWorkflow(WorkflowExecuteRunnable workflowExecuteThread) {
-        ProcessInstanceMetrics.incProcessInstanceSubmit();
-        submit(workflowExecuteThread);
-    }
-
     /**
      * Handle the events belong to the given workflow.
      */
@@ -139,7 +131,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
                 try {
                     LoggerUtils.setWorkflowInstanceIdMDC(workflowExecuteThread.getProcessInstance().getId());
                     if (workflowExecuteThread.workFlowFinish()) {
-                        stateWheelExecuteThread.removeProcess4TimeoutCheck(workflowExecuteThread.getProcessInstance());
+                        stateWheelExecuteThread.removeProcess4TimeoutCheck(workflowExecuteThread.getProcessInstance().getId());
                         processInstanceExecCacheManager.removeByProcessInstanceId(processInstanceId);
                         notifyProcessChanged(workflowExecuteThread.getProcessInstance());
                         logger.info("Workflow instance is finished.");
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
index 38fef8f44f..12c3332961 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
@@ -89,6 +89,8 @@ import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConst
 import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TARGET_CONNECTOR_TYPE;
 import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TARGET_DATASOURCE_ID;
 
+import lombok.NonNull;
+
 public abstract class BaseTaskProcessor implements ITaskProcessor {
 
     protected final Logger logger = LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT, getClass()));
@@ -107,22 +109,19 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
 
     protected int commitInterval;
 
-    protected ProcessService processService = SpringApplicationContext.getBean(ProcessService.class);
+    protected ProcessService processService;
 
-    protected MasterConfig masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
+    protected MasterConfig masterConfig;
 
-    protected TaskPluginManager taskPluginManager = SpringApplicationContext.getBean(TaskPluginManager.class);
+    protected TaskPluginManager taskPluginManager;
 
     protected String threadLoggerInfoName;
 
     @Override
-    public void init(TaskInstance taskInstance, ProcessInstance processInstance) {
-        if (processService == null) {
-            processService = SpringApplicationContext.getBean(ProcessService.class);
-        }
-        if (masterConfig == null) {
-            masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
-        }
+    public void init(@NonNull TaskInstance taskInstance, @NonNull ProcessInstance processInstance) {
+        processService = SpringApplicationContext.getBean(ProcessService.class);
+        masterConfig = SpringApplicationContext.getBean(MasterConfig.class);
+        taskPluginManager = SpringApplicationContext.getBean(TaskPluginManager.class);
         this.taskInstance = taskInstance;
         this.processInstance = processInstance;
         this.maxRetryTimes = masterConfig.getTaskCommitRetryTimes();
@@ -227,7 +226,7 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
 
     @Override
     public String getType() {
-        return null;
+        throw new UnsupportedOperationException("This abstract class doesn's has type");
     }
 
     @Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java
index 5b3d788c8c..d9ca82a45c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskInstanceKey.java
@@ -20,64 +20,22 @@ package org.apache.dolphinscheduler.server.master.runner.task;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 
-import java.util.Objects;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NonNull;
 
 /**
- * task instance key, processInstanceId
+ * Used to identify a task instance.
  */
+@Data
+@AllArgsConstructor
 public class TaskInstanceKey {
-    private int processInstanceId;
-    private long taskCode;
-    private int taskVersion;
+    private final int processInstanceId;
+    private final long taskCode;
+    private final int taskVersion;
 
-    public TaskInstanceKey(int processInstanceId, long taskCode, int taskVersion) {
-        this.processInstanceId = processInstanceId;
-        this.taskCode = taskCode;
-        this.taskVersion = taskVersion;
-    }
-
-    public int getProcessInstanceId() {
-        return processInstanceId;
-    }
-
-    public long getTaskCode() {
-        return taskCode;
-    }
-
-    public int getTaskVersion() {
-        return taskVersion;
-    }
-
-    public static TaskInstanceKey getTaskInstanceKey(ProcessInstance processInstance, TaskInstance taskInstance) {
-        if (processInstance == null || taskInstance == null) {
-            return null;
-        }
+    public static TaskInstanceKey getTaskInstanceKey(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
         return new TaskInstanceKey(processInstance.getId(), taskInstance.getTaskCode(), taskInstance.getTaskDefinitionVersion());
     }
 
-    @Override
-    public String toString() {
-        return "TaskKey{"
-                + "processInstanceId=" + processInstanceId
-                + ", taskCode=" + taskCode
-                + ", taskVersion=" + taskVersion
-                + '}';
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        TaskInstanceKey taskInstanceKey = (TaskInstanceKey) o;
-        return processInstanceId == taskInstanceKey.processInstanceId && taskCode == taskInstanceKey.taskCode && taskVersion == taskInstanceKey.taskVersion;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(processInstanceId, taskCode, taskVersion);
-    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
index 94f33ccd83..663cba4d90 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
@@ -17,19 +17,19 @@
 
 package org.apache.dolphinscheduler.server.master.service;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.NodeType;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.model.Server;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
 import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
@@ -49,6 +49,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,6 +57,7 @@ import org.springframework.stereotype.Component;
 
 import io.micrometer.core.annotation.Counted;
 import io.micrometer.core.annotation.Timed;
+import lombok.NonNull;
 
 /**
  * failover service
@@ -67,15 +69,20 @@ public class FailoverService {
     private final MasterConfig masterConfig;
     private final ProcessService processService;
     private final WorkflowExecuteThreadPool workflowExecuteThreadPool;
-
-    public FailoverService(RegistryClient registryClient,
-                           MasterConfig masterConfig,
-                           ProcessService processService,
-                           WorkflowExecuteThreadPool workflowExecuteThreadPool) {
-        this.registryClient = checkNotNull(registryClient);
-        this.masterConfig = checkNotNull(masterConfig);
-        this.processService = checkNotNull(processService);
-        this.workflowExecuteThreadPool = checkNotNull(workflowExecuteThreadPool);
+    private final ProcessInstanceExecCacheManager cacheManager;
+    private final String localAddress;
+
+    public FailoverService(@NonNull RegistryClient registryClient,
+                           @NonNull MasterConfig masterConfig,
+                           @NonNull ProcessService processService,
+                           @NonNull WorkflowExecuteThreadPool workflowExecuteThreadPool,
+                           @NonNull ProcessInstanceExecCacheManager cacheManager) {
+        this.registryClient = registryClient;
+        this.masterConfig = masterConfig;
+        this.processService = processService;
+        this.workflowExecuteThreadPool = workflowExecuteThreadPool;
+        this.cacheManager = cacheManager;
+        this.localAddress = NetUtils.getAddr(masterConfig.getListenPort());
     }
 
     /**
@@ -88,7 +95,7 @@ public class FailoverService {
         if (CollectionUtils.isEmpty(hosts)) {
             return;
         }
-        LOGGER.info("Master failover service {} begin to failover hosts:{}", getLocalAddress(), hosts);
+        LOGGER.info("Master failover service {} begin to failover hosts:{}", localAddress, hosts);
 
         for (String host : hosts) {
             failoverMasterWithLock(host);
@@ -174,11 +181,10 @@ public class FailoverService {
     }
 
     /**
-     * failover worker tasks
+     * Do the worker failover. Will find the SUBMITTED_SUCCESS/DISPATCH/RUNNING_EXECUTION/DELAY_EXECUTION/READY_PAUSE/READY_STOP tasks belong the given worker,
+     * and failover these tasks.
      * <p>
-     * 1. kill yarn job if there are yarn jobs in tasks.
-     * 2. change task state from running to need failover.
-     * 3. failover all tasks when workerHost is null
+     * Note: When we do worker failover, the master will only failover the processInstance belongs to the current master.
      *
      * @param workerHost worker host
      */
@@ -188,29 +194,40 @@ public class FailoverService {
         }
 
         long startTime = System.currentTimeMillis();
-        List<TaskInstance> needFailoverTaskInstanceList = processService.queryNeedFailoverTaskInstances(workerHost);
-        Map<Integer, ProcessInstance> processInstanceCacheMap = new HashMap<>();
+        // we query the task instance from cache, so that we can directly update the cache
+        final List<TaskInstance> needFailoverTaskInstanceList = cacheManager.getAll()
+            .stream()
+            .flatMap(workflowExecuteRunnable -> workflowExecuteRunnable.getAllTaskInstances().stream())
+            .filter(taskInstance ->
+                workerHost.equals(taskInstance.getHost()) && ExecutionStatus.isNeedFailoverWorkflowInstanceState(taskInstance.getState()))
+            .collect(Collectors.toList());
+        final Map<Integer, ProcessInstance> processInstanceCacheMap = new HashMap<>();
         LOGGER.info("start worker[{}] failover, task list size:{}", workerHost, needFailoverTaskInstanceList.size());
-        List<Server> workerServers = registryClient.getServerList(NodeType.WORKER);
+        final List<Server> workerServers = registryClient.getServerList(NodeType.WORKER);
         for (TaskInstance taskInstance : needFailoverTaskInstanceList) {
-            ProcessInstance processInstance = processInstanceCacheMap.get(taskInstance.getProcessInstanceId());
-            if (processInstance == null) {
-                processInstance = processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId());
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskInstance.getProcessInstanceId(), taskInstance.getId());
+            try {
+                ProcessInstance processInstance = processInstanceCacheMap.get(taskInstance.getProcessInstanceId());
                 if (processInstance == null) {
-                    LOGGER.error("failover task instance error, processInstance {} of taskInstance {} is null",
-                        taskInstance.getProcessInstanceId(), taskInstance.getId());
+                    processInstance = cacheManager.getByProcessInstanceId(taskInstance.getProcessInstanceId()).getProcessInstance();
+                    if (processInstance == null) {
+                        LOGGER.error("failover task instance error, processInstance {} of taskInstance {} is null",
+                            taskInstance.getProcessInstanceId(), taskInstance.getId());
+                        continue;
+                    }
+                    processInstanceCacheMap.put(processInstance.getId(), processInstance);
+                }
+
+                // only failover the task owned myself if worker down.
+                if (!StringUtils.equalsIgnoreCase(processInstance.getHost(), localAddress)) {
                     continue;
                 }
-                processInstanceCacheMap.put(processInstance.getId(), processInstance);
-            }
 
-            // only failover the task owned myself if worker down.
-            if (!processInstance.getHost().equalsIgnoreCase(getLocalAddress())) {
-                continue;
+                LOGGER.info("failover task instance id: {}, process instance id: {}", taskInstance.getId(), taskInstance.getProcessInstanceId());
+                failoverTaskInstance(processInstance, taskInstance, workerServers);
+            } finally {
+                LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
             }
-
-            LOGGER.info("failover task instance id: {}, process instance id: {}", taskInstance.getId(), taskInstance.getProcessInstanceId());
-            failoverTaskInstance(processInstance, taskInstance, workerServers);
         }
         LOGGER.info("end worker[{}] failover, useTime:{}ms", workerHost, System.currentTimeMillis() - startTime);
     }
@@ -221,17 +238,14 @@ public class FailoverService {
      * 1. kill yarn job if run on worker and there are yarn jobs in tasks.
      * 2. change task state from running to need failover.
      * 3. try to notify local master
+     *
      * @param processInstance
      * @param taskInstance
-     * @param servers if failover master, servers container master servers and worker servers; if failover worker, servers contain worker servers.
+     * @param servers         if failover master, servers container master servers and worker servers; if failover worker, servers contain worker servers.
      */
-    private void failoverTaskInstance(ProcessInstance processInstance, TaskInstance taskInstance, List<Server> servers) {
-        if (processInstance == null) {
-            LOGGER.error("failover task instance error, processInstance {} of taskInstance {} is null",
-                taskInstance.getProcessInstanceId(), taskInstance.getId());
-            return;
-        }
+    private void failoverTaskInstance(@NonNull ProcessInstance processInstance, TaskInstance taskInstance, List<Server> servers) {
         if (!checkTaskInstanceNeedFailover(servers, taskInstance)) {
+            LOGGER.info("The taskInstance doesn't need to failover");
             return;
         }
         TaskMetrics.incTaskFailover();
@@ -240,6 +254,7 @@ public class FailoverService {
         taskInstance.setProcessInstance(processInstance);
 
         if (!isMasterTask) {
+            LOGGER.info("The failover taskInstance is not master task");
             TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
                 .buildTaskInstanceRelatedInfo(taskInstance)
                 .buildProcessInstanceRelatedInfo(processInstance)
@@ -249,6 +264,8 @@ public class FailoverService {
                 // only kill yarn job if exists , the local thread has exited
                 ProcessUtils.killYarnJob(taskExecutionContext);
             }
+        } else {
+            LOGGER.info("The failover taskInstance is a master task");
         }
 
         taskInstance.setState(ExecutionStatus.NEED_FAULT_TOLERANCE);
@@ -278,7 +295,7 @@ public class FailoverService {
         while (iterator.hasNext()) {
             String host = iterator.next();
             if (registryClient.checkNodeExists(host, NodeType.MASTER)) {
-                if (!getLocalAddress().equals(host)) {
+                if (!localAddress.equals(host)) {
                     iterator.remove();
                 }
             }
@@ -390,11 +407,8 @@ public class FailoverService {
         return serverStartupTime;
     }
 
-    /**
-     * get local address
-     */
-    String getLocalAddress() {
-        return NetUtils.getAddr(masterConfig.getListenPort());
+    public String getLocalAddress() {
+        return localAddress;
     }
 
 }
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index a6c70f9871..65163cdb3c 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -92,7 +92,7 @@ master:
   pre-exec-threads: 10
   # master execute thread number to limit process instances in parallel
   exec-threads: 100
-  # master dispatch task number per batch
+  # master dispatch task number per batch, if all the tasks dispatch failed in a batch, will sleep 1s.
   dispatch-task-number: 3
   # master host selector to select a suitable worker, default value: LowerWeight. Optional values include random, round_robin, lower_weight
   host-selector: lower_weight
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
index d7b53f3c34..96bdfe4914 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/cache/impl/ProcessInstanceExecCacheManagerImplTest.java
@@ -56,7 +56,7 @@ public class ProcessInstanceExecCacheManagerImplTest {
         Assert.assertTrue(processInstanceExecCacheManager.contains(1));
     }
 
-    @Test
+    @Test(expected = NullPointerException.class)
     public void testCacheNull() {
         processInstanceExecCacheManager.cache(2, null);
         WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(2);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
index 06265d2e35..b2c7d63cd7 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutionContextTestUtils.java
@@ -49,8 +49,7 @@ public class ExecutionContextTestUtils {
                 .buildProcessDefinitionRelatedInfo(processDefinition)
                 .create();
 
-        TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand();
-        requestCommand.setTaskExecutionContext(JSONUtils.toJsonString(context));
+        TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand(context);
         Command command = requestCommand.convert2Command();
 
         ExecutionContext executionContext = new ExecutionContext(command, ExecutorType.WORKER);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
index 88c6655d2a..fdd79552b1 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java
@@ -97,7 +97,7 @@ public class NettyExecutorManagerTest {
 
     private Command toCommand(TaskExecutionContext taskExecutionContext) {
         TaskExecuteRequestCommand requestCommand = new TaskExecuteRequestCommand();
-        requestCommand.setTaskExecutionContext(JSONUtils.toJsonString(taskExecutionContext));
+        requestCommand.setTaskExecutionContext(taskExecutionContext);
         return requestCommand.convert2Command();
     }
 }
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
index 4e3d99347a..fc9bcca3aa 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/service/FailoverServiceTest.java
@@ -18,8 +18,8 @@
 package org.apache.dolphinscheduler.server.master.service;
 
 import static org.apache.dolphinscheduler.common.Constants.COMMON_TASK_TYPE;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DEPENDENT;
 import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_SWITCH;
+
 import static org.mockito.BDDMockito.given;
 import static org.mockito.Mockito.doNothing;
 
@@ -31,7 +31,9 @@ import org.apache.dolphinscheduler.common.model.Server;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteThreadPool;
 import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
 import org.apache.dolphinscheduler.service.process.ProcessService;
@@ -45,7 +47,6 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
@@ -63,7 +64,6 @@ import com.google.common.collect.Lists;
 @PrepareForTest({RegistryClient.class})
 @PowerMockIgnore({"javax.management.*"})
 public class FailoverServiceTest {
-    @InjectMocks
     private FailoverService failoverService;
 
     @Mock
@@ -78,6 +78,9 @@ public class FailoverServiceTest {
     @Mock
     private WorkflowExecuteThreadPool workflowExecuteThreadPool;
 
+    @Mock
+    private ProcessInstanceExecCacheManager cacheManager;
+
     private static int masterPort = 5678;
     private static int workerPort = 1234;
 
@@ -95,6 +98,7 @@ public class FailoverServiceTest {
         springApplicationContext.setApplicationContext(applicationContext);
 
         given(masterConfig.getListenPort()).willReturn(masterPort);
+        failoverService = new FailoverService(registryClient, masterConfig, processService, workflowExecuteThreadPool, cacheManager);
 
         testMasterHost = failoverService.getLocalAddress();
         String ip = testMasterHost.split(":")[0];
@@ -182,7 +186,16 @@ public class FailoverServiceTest {
 
     @Test
     public void failoverWorkTest() {
+        workerTaskInstance.setState(ExecutionStatus.RUNNING_EXECUTION);
+        WorkflowExecuteRunnable workflowExecuteRunnable = Mockito.mock(WorkflowExecuteRunnable.class);
+        Mockito.when(workflowExecuteRunnable.getAllTaskInstances()).thenReturn(Lists.newArrayList(workerTaskInstance));
+        Mockito.when(workflowExecuteRunnable.getProcessInstance()).thenReturn(processInstance);
+
+        Mockito.when(cacheManager.getAll()).thenReturn(Lists.newArrayList(workflowExecuteRunnable));
+        Mockito.when(cacheManager.getByProcessInstanceId(Mockito.anyInt())).thenReturn(workflowExecuteRunnable);
+
+
         failoverService.failoverServerWhenDown(testWorkerHost, NodeType.WORKER);
-        Assert.assertEquals(workerTaskInstance.getState(), ExecutionStatus.NEED_FAULT_TOLERANCE);
+        Assert.assertEquals(ExecutionStatus.NEED_FAULT_TOLERANCE, workerTaskInstance.getState());
     }
 }
diff --git a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java
index b7aecf7ee6..64259f2f4f 100644
--- a/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java
+++ b/dolphinscheduler-meter/src/main/java/org/apache/dolphinscheduler/meter/MeterConfiguration.java
@@ -20,10 +20,6 @@
 
 package org.apache.dolphinscheduler.meter;
 
-import javax.annotation.PostConstruct;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java
index 9baa321a9e..527a269411 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/Command.java
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.dolphinscheduler.remote.command;
 
 import java.io.Serializable;
@@ -25,6 +26,8 @@ import java.util.concurrent.atomic.AtomicLong;
  */
 public class Command implements Serializable {
 
+    private static final long serialVersionUID = -1L;
+
     private static final AtomicLong REQUEST_ID = new AtomicLong(1);
 
     public static final byte MAGIC = (byte) 0xbabe;
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java
index 5b2e33922c..000e3f4e02 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/TaskExecuteRequestCommand.java
@@ -18,39 +18,23 @@
 package org.apache.dolphinscheduler.remote.command;
 
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 
 import java.io.Serializable;
 
-/**
- *  execute task request command
- */
-public class TaskExecuteRequestCommand implements Serializable {
-
-    /**
-     *  task execution context
-     */
-    private String taskExecutionContext;
-
-    public String getTaskExecutionContext() {
-        return taskExecutionContext;
-    }
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
 
-    public void setTaskExecutionContext(String taskExecutionContext) {
-        this.taskExecutionContext = taskExecutionContext;
-    }
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class TaskExecuteRequestCommand implements Serializable {
 
-    public TaskExecuteRequestCommand() {
-    }
+    private static final long serialVersionUID = -1L;
 
-    public TaskExecuteRequestCommand(String taskExecutionContext) {
-        this.taskExecutionContext = taskExecutionContext;
-    }
+    private TaskExecutionContext taskExecutionContext;
 
-    /**
-     *  package request command
-     *
-     * @return command
-     */
     public Command convert2Command() {
         Command command = new Command();
         command.setType(CommandType.TASK_EXECUTE_REQUEST);
@@ -59,10 +43,4 @@ public class TaskExecuteRequestCommand implements Serializable {
         return command;
     }
 
-    @Override
-    public String toString() {
-        return "TaskExecuteRequestCommand{"
-                + "taskExecutionContext='" + taskExecutionContext + '\''
-                + '}';
-    }
 }
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
index 4dff8ede58..4c5ed4a3bc 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
@@ -44,6 +44,8 @@ import java.util.regex.Pattern;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import lombok.NonNull;
+
 /**
  * mainly used to get the start command line of a process.
  */
@@ -182,7 +184,10 @@ public class ProcessUtils {
      * @param taskExecutionContext taskExecutionContext
      * @return yarn application ids
      */
-    public static List<String> killYarnJob(TaskExecutionContext taskExecutionContext) {
+    public static List<String> killYarnJob(@NonNull TaskExecutionContext taskExecutionContext) {
+        if (taskExecutionContext.getLogPath() == null) {
+            return Collections.emptyList();
+        }
         try {
             Thread.sleep(Constants.SLEEP_TIME_MILLIS);
             String log;
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index bf48ab9cea..d6131174b3 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -52,16 +52,16 @@ import org.apache.dolphinscheduler.dao.entity.User;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.plugin.task.api.model.DateInterval;
 import org.apache.dolphinscheduler.spi.enums.ResourceType;
-import org.slf4j.Logger;
-import org.springframework.transaction.annotation.Transactional;
 
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
+import org.springframework.transaction.annotation.Transactional;
+
 public interface ProcessService {
     @Transactional
-    ProcessInstance handleCommand(Logger logger, String host, Command command);
+    ProcessInstance handleCommand(String host, Command command);
 
     void moveToErrorCommand(Command command, String message);
 
@@ -161,8 +161,6 @@ public interface ProcessService {
 
     void changeOutParam(TaskInstance taskInstance);
 
-    List<String> convertIntListToString(List<Integer> intList);
-
     Schedule querySchedule(int id);
 
     List<Schedule> queryReleaseSchedulerListByProcessDefinitionCode(long processDefinitionCode);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 4bc9de32f2..7021f4aeef 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -17,7 +17,6 @@
 
 package org.apache.dolphinscheduler.service.process;
 
-import static java.util.stream.Collectors.toSet;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS;
@@ -31,6 +30,8 @@ import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR
 import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
 import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TASK_INSTANCE_ID;
 
+import static java.util.stream.Collectors.toSet;
+
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.AuthorizationType;
 import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -170,13 +171,6 @@ public class ProcessServiceImpl implements ProcessService {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
 
-    private final int[] stateArray = new int[] {ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
-        ExecutionStatus.DISPATCH.ordinal(),
-        ExecutionStatus.RUNNING_EXECUTION.ordinal(),
-        ExecutionStatus.DELAY_EXECUTION.ordinal(),
-        ExecutionStatus.READY_PAUSE.ordinal(),
-        ExecutionStatus.READY_STOP.ordinal()};
-
     @Autowired
     private UserMapper userMapper;
 
@@ -281,7 +275,7 @@ public class ProcessServiceImpl implements ProcessService {
      */
     @Override
     @Transactional
-    public ProcessInstance handleCommand(Logger logger, String host, Command command) {
+    public ProcessInstance handleCommand(String host, Command command) {
         ProcessInstance processInstance = constructProcessInstance(command, host);
         // cannot construct process instance, return null
         if (processInstance == null) {
@@ -1947,8 +1941,7 @@ public class ProcessServiceImpl implements ProcessService {
      * @param intList intList
      * @return string list
      */
-    @Override
-    public List<String> convertIntListToString(List<Integer> intList) {
+    private List<String> convertIntListToString(List<Integer> intList) {
         if (intList == null) {
             return new ArrayList<>();
         }
@@ -2013,12 +2006,12 @@ public class ProcessServiceImpl implements ProcessService {
      */
     @Override
     public List<ProcessInstance> queryNeedFailoverProcessInstances(String host) {
-        return processInstanceMapper.queryByHostAndStatus(host, stateArray);
+        return processInstanceMapper.queryByHostAndStatus(host, ExecutionStatus.getNeedFailoverWorkflowInstanceState());
     }
 
     @Override
     public List<String> queryNeedFailoverProcessInstanceHost() {
-        return processInstanceMapper.queryNeedFailoverProcessInstanceHost(stateArray);
+        return processInstanceMapper.queryNeedFailoverProcessInstanceHost(ExecutionStatus.getNeedFailoverWorkflowInstanceState());
     }
 
     /**
@@ -2055,7 +2048,7 @@ public class ProcessServiceImpl implements ProcessService {
     @Override
     public List<TaskInstance> queryNeedFailoverTaskInstances(String host) {
         return taskInstanceMapper.queryByHostAndStatus(host,
-            stateArray);
+            ExecutionStatus.getNeedFailoverWorkflowInstanceState());
     }
 
     /**
@@ -2189,7 +2182,7 @@ public class ProcessServiceImpl implements ProcessService {
         return processInstanceMapper.queryLastRunningProcess(definitionCode,
             startTime,
             endTime,
-            stateArray);
+            ExecutionStatus.getNeedFailoverWorkflowInstanceState());
     }
 
     /**
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
index 47bdca17d1..e19a992b83 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
@@ -17,7 +17,12 @@
 
 package org.apache.dolphinscheduler.service.process;
 
-import com.fasterxml.jackson.databind.JsonNode;
+import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
+import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS;
+import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
+
+import static org.mockito.ArgumentMatchers.any;
+
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.CommandType;
 import org.apache.dolphinscheduler.common.enums.Flag;
@@ -74,6 +79,14 @@ import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
 import org.apache.dolphinscheduler.service.exceptions.ServiceException;
 import org.apache.dolphinscheduler.service.quartz.cron.CronUtilsTest;
 import org.apache.dolphinscheduler.spi.params.base.FormType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -87,17 +100,7 @@ import org.powermock.reflect.Whitebox;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
-import static org.mockito.ArgumentMatchers.any;
+import com.fasterxml.jackson.databind.JsonNode;
 
 /**
  * process service test
@@ -284,9 +287,12 @@ public class ProcessServiceTest {
         Command command = new Command();
         command.setProcessDefinitionCode(222);
         command.setCommandType(CommandType.REPEAT_RUNNING);
-        command.setCommandParam("{\"" + CMD_PARAM_RECOVER_PROCESS_ID_STRING + "\":\"111\",\""
-                + CMD_PARAM_SUB_PROCESS_DEFINE_CODE + "\":\"222\"}");
-        Assert.assertNull(processService.handleCommand(logger, host, command));
+        command.setCommandParam("{\""
+                                    + CMD_PARAM_RECOVER_PROCESS_ID_STRING
+                                    + "\":\"111\",\""
+                                    + CMD_PARAM_SUB_PROCESS_DEFINE_CODE
+                                    + "\":\"222\"}");
+        Assert.assertNull(processService.handleCommand(host, command));
 
         int definitionVersion = 1;
         long definitionCode = 123;
@@ -321,7 +327,7 @@ public class ProcessServiceTest {
         Mockito.when(processDefineLogMapper.queryByDefinitionCodeAndVersion(processInstance.getProcessDefinitionCode(),
                 processInstance.getProcessDefinitionVersion())).thenReturn(new ProcessDefinitionLog(processDefinition));
         Mockito.when(processInstanceMapper.queryDetailById(222)).thenReturn(processInstance);
-        Assert.assertNotNull(processService.handleCommand(logger, host, command1));
+        Assert.assertNotNull(processService.handleCommand(host, command1));
 
         Command command2 = new Command();
         command2.setId(2);
@@ -331,7 +337,7 @@ public class ProcessServiceTest {
         command2.setCommandType(CommandType.RECOVER_SUSPENDED_PROCESS);
         command2.setProcessInstanceId(processInstanceId);
         Mockito.when(commandMapper.deleteById(2)).thenReturn(1);
-        Assert.assertNotNull(processService.handleCommand(logger, host, command2));
+        Assert.assertNotNull(processService.handleCommand(host, command2));
 
         Command command3 = new Command();
         command3.setId(3);
@@ -341,7 +347,7 @@ public class ProcessServiceTest {
         command3.setCommandParam("{\"WaitingThreadInstanceId\":222}");
         command3.setCommandType(CommandType.START_FAILURE_TASK_PROCESS);
         Mockito.when(commandMapper.deleteById(3)).thenReturn(1);
-        Assert.assertNotNull(processService.handleCommand(logger, host, command3));
+        Assert.assertNotNull(processService.handleCommand(host, command3));
 
         Command command4 = new Command();
         command4.setId(4);
@@ -351,7 +357,7 @@ public class ProcessServiceTest {
         command4.setCommandType(CommandType.REPEAT_RUNNING);
         command4.setProcessInstanceId(processInstanceId);
         Mockito.when(commandMapper.deleteById(4)).thenReturn(1);
-        Assert.assertNotNull(processService.handleCommand(logger, host, command4));
+        Assert.assertNotNull(processService.handleCommand(host, command4));
 
         Command command5 = new Command();
         command5.setId(5);
@@ -390,7 +396,7 @@ public class ProcessServiceTest {
         Mockito.when(processInstanceMapper.queryDetailById(223)).thenReturn(processInstance2);
         Mockito.when(processDefineMapper.queryByCode(11L)).thenReturn(processDefinition1);
         Mockito.when(commandMapper.deleteById(1)).thenReturn(1);
-        Assert.assertNotNull(processService.handleCommand(logger, host, command1));
+        Assert.assertNotNull(processService.handleCommand(host, command1));
 
         Command command6 = new Command();
         command6.setId(6);
@@ -401,7 +407,7 @@ public class ProcessServiceTest {
         Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(11L, 1, Constants.RUNNING_PROCESS_STATE, 223)).thenReturn(lists);
         Mockito.when(processInstanceMapper.updateNextProcessIdById(223, 222)).thenReturn(true);
         Mockito.when(commandMapper.deleteById(6)).thenReturn(1);
-        ProcessInstance processInstance6 = processService.handleCommand(logger, host, command6);
+        ProcessInstance processInstance6 = processService.handleCommand(host, command6);
         Assert.assertTrue(processInstance6 != null);
 
         processDefinition1.setExecutionType(ProcessExecutionTypeEnum.SERIAL_DISCARD);
@@ -444,7 +450,7 @@ public class ProcessServiceTest {
         Mockito.when(processInstanceMapper.queryByProcessDefineCodeAndProcessDefinitionVersionAndStatusAndNextId(12L, 1, Constants.RUNNING_PROCESS_STATE, 0)).thenReturn(lists);
         Mockito.when(processInstanceMapper.updateById(processInstance)).thenReturn(1);
         Mockito.when(commandMapper.deleteById(9)).thenReturn(1);
-        ProcessInstance processInstance10 = processService.handleCommand(logger, host, command9);
+        ProcessInstance processInstance10 = processService.handleCommand(host, command9);
         Assert.assertTrue(processInstance10 == null);
     }
 
@@ -485,7 +491,7 @@ public class ProcessServiceTest {
         Mockito.when(processInstanceMapper.queryDetailById(222)).thenReturn(processInstance);
 
         // will throw exception when command id is 0 and delete fail
-        processService.handleCommand(logger, host, command1);
+        processService.handleCommand(host, command1);
     }
 
     @Test
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java
index 581c73c8f1..0ae6f31c35 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java
@@ -22,13 +22,25 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy;
 import org.apache.dolphinscheduler.plugin.task.api.model.Property;
 import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
 
+import java.io.Serializable;
 import java.util.Date;
 import java.util.Map;
 
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
 /**
  * to master/worker task transport
  */
-public class TaskExecutionContext {
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class TaskExecutionContext implements Serializable {
+
+    private static final long serialVersionUID = -1L;
 
     /**
      * task id
@@ -236,373 +248,4 @@ public class TaskExecutionContext {
 
     private DataQualityTaskExecutionContext dataQualityTaskExecutionContext;
 
-    public String getTaskLogName() {
-        return taskLogName;
-    }
-
-    public void setTaskLogName(String taskLogName) {
-        this.taskLogName = taskLogName;
-    }
-
-    public Map<String, String> getResources() {
-        return resources;
-    }
-
-    public void setResources(Map<String, String> resources) {
-        this.resources = resources;
-    }
-
-    public Map<String, Property> getParamsMap() {
-        return paramsMap;
-    }
-
-    public void setParamsMap(Map<String, Property> paramsMap) {
-        this.paramsMap = paramsMap;
-    }
-
-    public int getTaskInstanceId() {
-        return taskInstanceId;
-    }
-
-    public void setTaskInstanceId(int taskInstanceId) {
-        this.taskInstanceId = taskInstanceId;
-    }
-
-    public String getTaskName() {
-        return taskName;
-    }
-
-    public void setTaskName(String taskName) {
-        this.taskName = taskName;
-    }
-
-    public Date getFirstSubmitTime() {
-        return firstSubmitTime;
-    }
-
-    public void setFirstSubmitTime(Date firstSubmitTime) {
-        this.firstSubmitTime = firstSubmitTime;
-    }
-
-    public Date getStartTime() {
-        return startTime;
-    }
-
-    public void setStartTime(Date startTime) {
-        this.startTime = startTime;
-    }
-
-    public String getTaskType() {
-        return taskType;
-    }
-
-    public void setTaskType(String taskType) {
-        this.taskType = taskType;
-    }
-
-    public String getHost() {
-        return host;
-    }
-
-    public void setHost(String host) {
-        this.host = host;
-    }
-
-    public String getExecutePath() {
-        return executePath;
-    }
-
-    public void setExecutePath(String executePath) {
-        this.executePath = executePath;
-    }
-
-    public String getLogPath() {
-        return logPath;
-    }
-
-    public void setLogPath(String logPath) {
-        this.logPath = logPath;
-    }
-
-    public String getTaskJson() {
-        return taskJson;
-    }
-
-    public void setTaskJson(String taskJson) {
-        this.taskJson = taskJson;
-    }
-
-    public int getProcessId() {
-        return processId;
-    }
-
-    public void setProcessId(int processId) {
-        this.processId = processId;
-    }
-
-    public String getAppIds() {
-        return appIds;
-    }
-
-    public void setAppIds(String appIds) {
-        this.appIds = appIds;
-    }
-
-    public int getProcessInstanceId() {
-        return processInstanceId;
-    }
-
-    public void setProcessInstanceId(int processInstanceId) {
-        this.processInstanceId = processInstanceId;
-    }
-
-    public Date getScheduleTime() {
-        return scheduleTime;
-    }
-
-    public void setScheduleTime(Date scheduleTime) {
-        this.scheduleTime = scheduleTime;
-    }
-
-    public String getGlobalParams() {
-        return globalParams;
-    }
-
-    public void setGlobalParams(String globalParams) {
-        this.globalParams = globalParams;
-    }
-
-    public int getExecutorId() {
-        return executorId;
-    }
-
-    public void setExecutorId(int executorId) {
-        this.executorId = executorId;
-    }
-
-    public int getCmdTypeIfComplement() {
-        return cmdTypeIfComplement;
-    }
-
-    public void setCmdTypeIfComplement(int cmdTypeIfComplement) {
-        this.cmdTypeIfComplement = cmdTypeIfComplement;
-    }
-
-    public String getTenantCode() {
-        return tenantCode;
-    }
-
-    public void setTenantCode(String tenantCode) {
-        this.tenantCode = tenantCode;
-    }
-
-    public String getQueue() {
-        return queue;
-    }
-
-    public void setQueue(String queue) {
-        this.queue = queue;
-    }
-
-    public int getProcessDefineId() {
-        return processDefineId;
-    }
-
-    public void setProcessDefineId(int processDefineId) {
-        this.processDefineId = processDefineId;
-    }
-
-    public int getProjectId() {
-        return projectId;
-    }
-
-    public void setProjectId(int projectId) {
-        this.projectId = projectId;
-    }
-
-    public String getTaskParams() {
-        return taskParams;
-    }
-
-    public void setTaskParams(String taskParams) {
-        this.taskParams = taskParams;
-    }
-
-    public String getEnvFile() {
-        return envFile;
-    }
-
-    public void setEnvFile(String envFile) {
-        this.envFile = envFile;
-    }
-
-    public String getEnvironmentConfig() {
-        return environmentConfig;
-    }
-
-    public void setEnvironmentConfig(String config) {
-        this.environmentConfig = config;
-    }
-
-    public Map<String, String> getDefinedParams() {
-        return definedParams;
-    }
-
-    public void setDefinedParams(Map<String, String> definedParams) {
-        this.definedParams = definedParams;
-    }
-
-    public String getTaskAppId() {
-        return taskAppId;
-    }
-
-    public void setTaskAppId(String taskAppId) {
-        this.taskAppId = taskAppId;
-    }
-
-    public TaskTimeoutStrategy getTaskTimeoutStrategy() {
-        return taskTimeoutStrategy;
-    }
-
-    public void setTaskTimeoutStrategy(TaskTimeoutStrategy taskTimeoutStrategy) {
-        this.taskTimeoutStrategy = taskTimeoutStrategy;
-    }
-
-    public int getTaskTimeout() {
-        return taskTimeout;
-    }
-
-    public void setTaskTimeout(int taskTimeout) {
-        this.taskTimeout = taskTimeout;
-    }
-
-    public String getWorkerGroup() {
-        return workerGroup;
-    }
-
-    public void setWorkerGroup(String workerGroup) {
-        this.workerGroup = workerGroup;
-    }
-
-    public int getDelayTime() {
-        return delayTime;
-    }
-
-    public void setDelayTime(int delayTime) {
-        this.delayTime = delayTime;
-    }
-
-    public ResourceParametersHelper getResourceParametersHelper() {
-        return resourceParametersHelper;
-    }
-
-    public void setResourceParametersHelper(ResourceParametersHelper resourceParametersHelper) {
-        this.resourceParametersHelper = resourceParametersHelper;
-    }
-
-    public String getVarPool() {
-        return varPool;
-    }
-
-    public void setVarPool(String varPool) {
-        this.varPool = varPool;
-    }
-
-    public int getDryRun() {
-        return dryRun;
-    }
-
-    public void setDryRun(int dryRun) {
-        this.dryRun = dryRun;
-    }
-
-    public Long getProcessDefineCode() {
-        return processDefineCode;
-    }
-
-    public void setProcessDefineCode(Long processDefineCode) {
-        this.processDefineCode = processDefineCode;
-    }
-
-    public int getProcessDefineVersion() {
-        return processDefineVersion;
-    }
-
-    public void setProcessDefineVersion(int processDefineVersion) {
-        this.processDefineVersion = processDefineVersion;
-    }
-
-    public long getProjectCode() {
-        return projectCode;
-    }
-
-    public void setProjectCode(long projectCode) {
-        this.projectCode = projectCode;
-    }
-
-    public DataQualityTaskExecutionContext getDataQualityTaskExecutionContext() {
-        return dataQualityTaskExecutionContext;
-    }
-
-    public void setDataQualityTaskExecutionContext(DataQualityTaskExecutionContext dataQualityTaskExecutionContext) {
-        this.dataQualityTaskExecutionContext = dataQualityTaskExecutionContext;
-    }
-
-    public void setCurrentExecutionStatus(ExecutionStatus currentExecutionStatus) {
-        this.currentExecutionStatus = currentExecutionStatus;
-    }
-
-    public ExecutionStatus getCurrentExecutionStatus() {
-        return currentExecutionStatus;
-    }
-
-    public Date getEndTime() {
-        return endTime;
-    }
-
-    public void setEndTime(Date endTime) {
-        this.endTime = endTime;
-    }
-
-    @Override
-    public String toString() {
-        return "TaskExecutionContext{"
-                + "taskInstanceId=" + taskInstanceId
-                + ", taskName='" + taskName + '\''
-                + ", currentExecutionStatus=" + currentExecutionStatus
-                + ", firstSubmitTime=" + firstSubmitTime
-                + ", startTime=" + startTime
-                + ", taskType='" + taskType + '\''
-                + ", host='" + host + '\''
-                + ", executePath='" + executePath + '\''
-                + ", logPath='" + logPath + '\''
-                + ", taskJson='" + taskJson + '\''
-                + ", processId=" + processId
-                + ", processDefineCode=" + processDefineCode
-                + ", processDefineVersion=" + processDefineVersion
-                + ", appIds='" + appIds + '\''
-                + ", processInstanceId=" + processInstanceId
-                + ", scheduleTime=" + scheduleTime
-                + ", globalParams='" + globalParams + '\''
-                + ", executorId=" + executorId
-                + ", cmdTypeIfComplement=" + cmdTypeIfComplement
-                + ", tenantCode='" + tenantCode + '\''
-                + ", queue='" + queue + '\''
-                + ", projectCode=" + projectCode
-                + ", taskParams='" + taskParams + '\''
-                + ", envFile='" + envFile + '\''
-                + ", dryRun='" + dryRun + '\''
-                + ", definedParams=" + definedParams
-                + ", taskAppId='" + taskAppId + '\''
-                + ", taskTimeoutStrategy=" + taskTimeoutStrategy
-                + ", taskTimeout=" + taskTimeout
-                + ", workerGroup='" + workerGroup + '\''
-                + ", environmentConfig='" + environmentConfig + '\''
-                + ", delayTime=" + delayTime
-                + ", resources=" + resources
-                + ", sqlTaskExecutionContext=" + sqlTaskExecutionContext
-                + ", dataQualityTaskExecutionContext=" + dataQualityTaskExecutionContext
-                + '}';
-    }
-
 }
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java
index 82c8db12d4..411de30c34 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java
@@ -78,6 +78,15 @@ public enum ExecutionStatus {
 
     private static HashMap<Integer, ExecutionStatus> EXECUTION_STATUS_MAP = new HashMap<>();
 
+    private static final int[] NEED_FAILOVER_STATES = new int[] {
+        ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
+        ExecutionStatus.DISPATCH.ordinal(),
+        ExecutionStatus.RUNNING_EXECUTION.ordinal(),
+        ExecutionStatus.DELAY_EXECUTION.ordinal(),
+        ExecutionStatus.READY_PAUSE.ordinal(),
+        ExecutionStatus.READY_STOP.ordinal()
+    };
+
     static {
         for (ExecutionStatus executionStatus : ExecutionStatus.values()) {
             EXECUTION_STATUS_MAP.put(executionStatus.code, executionStatus);
@@ -180,4 +189,18 @@ public enum ExecutionStatus {
         }
         throw new IllegalArgumentException("invalid status : " + status);
     }
+
+    public static boolean isNeedFailoverWorkflowInstanceState(ExecutionStatus executionStatus) {
+        return
+            ExecutionStatus.SUBMITTED_SUCCESS == executionStatus
+                || ExecutionStatus.DISPATCH == executionStatus
+                || ExecutionStatus.RUNNING_EXECUTION == executionStatus
+                || ExecutionStatus.DELAY_EXECUTION == executionStatus
+                || ExecutionStatus.READY_PAUSE == executionStatus
+                || ExecutionStatus.READY_STOP == executionStatus;
+    }
+
+    public static int[] getNeedFailoverWorkflowInstanceState() {
+        return NEED_FAILOVER_STATES;
+    }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java
index 9613211f43..6c68cc00ec 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/metrics/WorkerServerMetrics.java
@@ -22,22 +22,20 @@ import java.util.function.Supplier;
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.Gauge;
 import io.micrometer.core.instrument.Metrics;
+import lombok.experimental.UtilityClass;
 
-public final class WorkerServerMetrics {
-
-    public WorkerServerMetrics() {
-        throw new UnsupportedOperationException("Utility class");
-    }
+@UtilityClass
+public class WorkerServerMetrics {
 
     private static final Counter WORKER_OVERLOAD_COUNTER =
-            Counter.builder("dolphinscheduler_worker_overload_count")
-                    .description("worker load count")
-                    .register(Metrics.globalRegistry);
+        Counter.builder("ds.worker.overload.count")
+            .description("overloaded workers count")
+            .register(Metrics.globalRegistry);
 
     private static final Counter WORKER_SUBMIT_QUEUE_IS_FULL_COUNTER =
-            Counter.builder("dolphinscheduler_worker_submit_queue_is_full_count")
-                    .description("worker task submit queue is full count")
-                    .register(Metrics.globalRegistry);
+        Counter.builder("ds.worker.full.submit.queue.count")
+            .description("full worker submit queues count")
+            .register(Metrics.globalRegistry);
 
     public static void incWorkerOverloadCount() {
         WORKER_OVERLOAD_COUNTER.increment();
@@ -48,9 +46,10 @@ public final class WorkerServerMetrics {
     }
 
     public static void registerWorkerRunningTaskGauge(Supplier<Number> supplier) {
-        Gauge.builder("dolphinscheduler_worker_running_task_gauge", supplier)
-                .description("worker running task gauge")
-                .register(Metrics.globalRegistry);
+        Gauge.builder("ds.task.running", supplier)
+            .description("number of running tasks on workers")
+            .register(Metrics.globalRegistry);
 
     }
+
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
index 5fff6a24df..5a9e2bcef6 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
@@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.remote.processor.NettyRemoteChannel;
 import org.apache.dolphinscheduler.server.worker.cache.ResponseCache;
 
 import java.util.Arrays;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.slf4j.Logger;
@@ -91,9 +92,6 @@ public class TaskCallbackService {
      * change remote channel
      */
     public void changeRemoteChannel(int taskInstanceId, NettyRemoteChannel channel) {
-        if (REMOTE_CHANNELS.containsKey(taskInstanceId)) {
-            REMOTE_CHANNELS.remove(taskInstanceId);
-        }
         REMOTE_CHANNELS.put(taskInstanceId, channel);
     }
 
@@ -103,19 +101,19 @@ public class TaskCallbackService {
      * @param taskInstanceId taskInstanceId
      * @return callback channel
      */
-    private NettyRemoteChannel getRemoteChannel(int taskInstanceId) {
+    private Optional<NettyRemoteChannel> getRemoteChannel(int taskInstanceId) {
         Channel newChannel;
         NettyRemoteChannel nettyRemoteChannel = REMOTE_CHANNELS.get(taskInstanceId);
         if (nettyRemoteChannel != null) {
             if (nettyRemoteChannel.isActive()) {
-                return nettyRemoteChannel;
+                return Optional.of(nettyRemoteChannel);
             }
             newChannel = nettyRemotingClient.getChannel(nettyRemoteChannel.getHost());
             if (newChannel != null) {
-                return getRemoteChannel(newChannel, nettyRemoteChannel.getOpaque(), taskInstanceId);
+                return Optional.of(getRemoteChannel(newChannel, nettyRemoteChannel.getOpaque(), taskInstanceId));
             }
         }
-        return null;
+        return Optional.empty();
     }
 
     public long pause(int ntries) {
@@ -150,15 +148,13 @@ public class TaskCallbackService {
      * @param command command
      */
     public void send(int taskInstanceId, Command command) {
-        NettyRemoteChannel nettyRemoteChannel = getRemoteChannel(taskInstanceId);
-        if (nettyRemoteChannel != null) {
-            nettyRemoteChannel.writeAndFlush(command).addListener(new ChannelFutureListener() {
-
+        Optional<NettyRemoteChannel> nettyRemoteChannel = getRemoteChannel(taskInstanceId);
+        if (nettyRemoteChannel.isPresent()) {
+            nettyRemoteChannel.get().writeAndFlush(command).addListener(new ChannelFutureListener() {
                 @Override
-                public void operationComplete(ChannelFuture future) throws Exception {
-                    if (future.isSuccess()) {
-                        // remove(taskInstanceId);
-                        return;
+                public void operationComplete(ChannelFuture future) {
+                    if (!future.isSuccess()) {
+                        logger.error("Send callback command error, taskInstanceId: {}, command: {}", taskInstanceId, command);
                     }
                 }
             });
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
index 484acf418e..45ebbc2439 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
@@ -109,8 +109,7 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
         }
         logger.info("task execute request command : {}", taskRequestCommand);
 
-        String contextJson = taskRequestCommand.getTaskExecutionContext();
-        TaskExecutionContext taskExecutionContext = JSONUtils.parseObject(contextJson, TaskExecutionContext.class);
+        TaskExecutionContext taskExecutionContext = taskRequestCommand.getTaskExecutionContext();
 
         if (taskExecutionContext == null) {
             logger.error("task execution context is null");
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
index 0e48465a07..108ae8a3bb 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.server.worker.runner;
 
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.remote.command.Command;
@@ -34,7 +35,7 @@ import org.springframework.stereotype.Component;
  * Retry Report Task Status Thread
  */
 @Component
-public class RetryReportTaskStatusThread implements Runnable {
+public class RetryReportTaskStatusThread extends BaseDaemonThread {
 
     private final Logger logger = LoggerFactory.getLogger(RetryReportTaskStatusThread.class);
 
@@ -46,11 +47,14 @@ public class RetryReportTaskStatusThread implements Runnable {
     @Autowired
     private TaskCallbackService taskCallbackService;
 
-    public void start() {
+    protected RetryReportTaskStatusThread() {
+        super("RetryReportTaskStatusThread");
+    }
+
+    @Override
+    public synchronized void start() {
         logger.info("Retry report task status thread starting");
-        Thread thread = new Thread(this, "RetryReportTaskStatusThread");
-        thread.setDaemon(true);
-        thread.start();
+        super.start();
         logger.info("Retry report task status thread started");
     }
 
@@ -59,7 +63,7 @@ public class RetryReportTaskStatusThread implements Runnable {
      */
     @Override
     public void run() {
-        ResponseCache instance = ResponseCache.get();
+        final ResponseCache instance = ResponseCache.get();
 
         while (Stopper.isRunning()) {
 
@@ -67,25 +71,45 @@ public class RetryReportTaskStatusThread implements Runnable {
             ThreadUtils.sleep(RETRY_REPORT_TASK_STATUS_INTERVAL);
 
             try {
-                if (!instance.getRunningCache().isEmpty()) {
-                    Map<Integer, Command> runningCache = instance.getRunningCache();
-                    for (Map.Entry<Integer, Command> entry : runningCache.entrySet()) {
-                        Integer taskInstanceId = entry.getKey();
-                        Command runningCommand = entry.getValue();
-                        taskCallbackService.send(taskInstanceId, runningCommand);
-                    }
+                // todo: Only retry the send failed command
+                retryRunningCommand(instance);
+                retryResponseCommand(instance);
+            } catch (Exception e) {
+                logger.warn("Retry report task status error", e);
+            }
+        }
+    }
+
+    private void retryRunningCommand(ResponseCache instance) {
+        if (!instance.getRunningCache().isEmpty()) {
+            Map<Integer, Command> runningCache = instance.getRunningCache();
+            for (Map.Entry<Integer, Command> entry : runningCache.entrySet()) {
+                Integer taskInstanceId = entry.getKey();
+                Command runningCommand = entry.getValue();
+                try {
+                    taskCallbackService.send(taskInstanceId, runningCommand);
+                } catch (Exception ex) {
+                    logger.error("Retry send running command to master error, taskInstanceId: {}, command: {}",
+                                 taskInstanceId,
+                                 runningCommand);
                 }
+            }
+        }
+    }
 
-                if (!instance.getResponseCache().isEmpty()) {
-                    Map<Integer, Command> responseCache = instance.getResponseCache();
-                    for (Map.Entry<Integer, Command> entry : responseCache.entrySet()) {
-                        Integer taskInstanceId = entry.getKey();
-                        Command responseCommand = entry.getValue();
-                        taskCallbackService.send(taskInstanceId, responseCommand);
-                    }
+    private void retryResponseCommand(ResponseCache instance) {
+        if (!instance.getResponseCache().isEmpty()) {
+            Map<Integer, Command> responseCache = instance.getResponseCache();
+            for (Map.Entry<Integer, Command> entry : responseCache.entrySet()) {
+                Integer taskInstanceId = entry.getKey();
+                Command responseCommand = entry.getValue();
+                try {
+                    taskCallbackService.send(taskInstanceId, responseCommand);
+                } catch (Exception ex) {
+                    logger.error("Retry send response command to master error, taskInstanceId: {}, command: {}",
+                                 taskInstanceId,
+                                 responseCommand);
                 }
-            } catch (Exception e) {
-                logger.warn("Retry report task status error", e);
             }
         }
     }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
index ae89e9e946..b5fecf6c02 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@@ -131,9 +131,10 @@ public class TaskExecuteThread implements Runnable, Delayed {
             taskExecutionContext.setEndTime(new Date());
             TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
             taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+            logger.info("[WorkflowInstance-{}][TaskInstance-{}] Task dry run success",
+                taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId());
             return;
         }
-
         try {
             LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId());
             logger.info("script path : {}", taskExecutionContext.getExecutePath());
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
index 13c533f1cf..2187a19281 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
@@ -88,7 +88,7 @@ public class TaskExecuteProcessorTest {
         command = new Command();
         command.setType(CommandType.TASK_EXECUTE_REQUEST);
         ackCommand = new TaskExecuteRunningCommand().convert2Command();
-        taskRequestCommand = new TaskExecuteRequestCommand();
+        taskRequestCommand = new TaskExecuteRequestCommand(taskExecutionContext);
         alertClientService = PowerMockito.mock(AlertClientService.class);
         workerExecService = PowerMockito.mock(ExecutorService.class);
         PowerMockito.when(workerExecService.submit(Mockito.any(TaskExecuteThread.class)))
@@ -127,8 +127,6 @@ public class TaskExecuteProcessorTest {
         PowerMockito.mockStatic(JSONUtils.class);
         PowerMockito.when(JSONUtils.parseObject(command.getBody(), TaskExecuteRequestCommand.class))
                 .thenReturn(taskRequestCommand);
-        PowerMockito.when(JSONUtils.parseObject(taskRequestCommand.getTaskExecutionContext(), TaskExecutionContext.class))
-                .thenReturn(taskExecutionContext);
 
         PowerMockito.mockStatic(FileUtils.class);
         PowerMockito.when(FileUtils.getProcessExecDir(taskExecutionContext.getProjectCode(),


[dolphinscheduler] 04/29: Add some warning log in master (#10383)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 52815975bc703adba741038e321567d8e8848913
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Fri Jun 10 19:38:20 2022 +0800

    Add some warning log in master (#10383)
    
    * Add some warn log in master
    
    * fix may skip sleep
    
    (cherry picked from commit b0d9d3f9ab20f822ce019eb73c893ad8811d14a9)
---
 .../server/master/config/MasterConfig.java         | 34 ++++++++++++++++++++++
 .../master/consumer/TaskPriorityQueueConsumer.java | 25 ++++++++--------
 .../master/processor/TaskRecallProcessor.java      |  0
 .../master/runner/MasterSchedulerService.java      | 13 ++++-----
 .../master/runner/StateWheelExecuteThread.java     | 10 +++++--
 .../master/runner/WorkflowExecuteRunnable.java     |  4 +--
 .../master/runner/WorkflowExecuteThreadPool.java   | 16 ++++++----
 .../remote/NettyRemotingClient.java                |  4 ++-
 .../service/process/ProcessServiceImpl.java        |  6 ++--
 .../service/queue/TaskPriorityQueueImpl.java       |  9 ++----
 10 files changed, 82 insertions(+), 39 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
index f7a8dddd03..76d4ae1525 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
@@ -18,6 +18,8 @@
 package org.apache.dolphinscheduler.server.master.config;
 
 import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostSelector;
+import org.apache.dolphinscheduler.server.master.processor.queue.TaskExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -27,15 +29,47 @@ import org.springframework.stereotype.Component;
 @EnableConfigurationProperties
 @ConfigurationProperties("master")
 public class MasterConfig {
+    /**
+     * The master RPC server listen port.
+     */
     private int listenPort;
+    /**
+     * The max batch size used to fetch command from database.
+     */
     private int fetchCommandNum;
+    /**
+     * The thread number used to prepare processInstance. This number shouldn't bigger than fetchCommandNum.
+     */
     private int preExecThreads;
+    /**
+     * todo: We may need to split the process/task into different thread size.
+     * The thread number used to handle processInstance and task event.
+     * Will create two thread poll to execute {@link WorkflowExecuteRunnable} and {@link TaskExecuteRunnable}.
+     */
     private int execThreads;
+    /**
+     * The task dispatch thread pool size.
+     */
     private int dispatchTaskNumber;
+    /**
+     * Worker select strategy.
+     */
     private HostSelector hostSelector;
+    /**
+     * Master heart beat task execute interval.
+     */
     private int heartbeatInterval;
+    /**
+     * task submit max retry times.
+     */
     private int taskCommitRetryTimes;
+    /**
+     * task submit retry interval/ms.
+     */
     private int taskCommitInterval;
+    /**
+     * state wheel check interval/ms, if this value is bigger, may increase the delay of task/processInstance.
+     */
     private int stateWheelInterval;
     private double maxCpuLoadAvg;
     private double reservedMemory;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index 0dbfb73a2c..d75595de1a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -39,6 +39,8 @@ import org.apache.dolphinscheduler.service.queue.TaskPriority;
 import org.apache.dolphinscheduler.service.queue.TaskPriorityQueue;
 import org.apache.dolphinscheduler.spi.utils.JSONUtils;
 
+import org.apache.commons.collections.CollectionUtils;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -54,11 +56,6 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
-import io.micrometer.core.annotation.Counted;
-import io.micrometer.core.annotation.Timed;
-
-import org.apache.commons.lang3.time.StopWatch;
-
 /**
  * TaskUpdateQueue consumer
  */
@@ -124,7 +121,7 @@ public class TaskPriorityQueueConsumer extends Thread {
             try {
                 List<TaskPriority> failedDispatchTasks = this.batchDispatch(fetchTaskNum);
 
-                if (!failedDispatchTasks.isEmpty()) {
+                if (CollectionUtils.isNotEmpty(failedDispatchTasks)) {
                     TaskMetrics.incTaskDispatchFailed(failedDispatchTasks.size());
                     for (TaskPriority dispatchFailedTask : failedDispatchTasks) {
                         taskPriorityQueue.put(dispatchFailedTask);
@@ -157,11 +154,15 @@ public class TaskPriorityQueueConsumer extends Thread {
             }
 
             consumerThreadPoolExecutor.submit(() -> {
-                boolean dispatchResult = this.dispatchTask(taskPriority);
-                if (!dispatchResult) {
-                    failedDispatchTasks.add(taskPriority);
+                try {
+                    boolean dispatchResult = this.dispatchTask(taskPriority);
+                    if (!dispatchResult) {
+                        failedDispatchTasks.add(taskPriority);
+                    }
+                } finally {
+                    // make sure the latch countDown
+                    latch.countDown();
                 }
-                latch.countDown();
             });
         }
 
@@ -171,10 +172,10 @@ public class TaskPriorityQueueConsumer extends Thread {
     }
 
     /**
-     * dispatch task
+     * Dispatch task to worker.
      *
      * @param taskPriority taskPriority
-     * @return result
+     * @return dispatch result, return true if dispatch success, return false if dispatch failed.
      */
     protected boolean dispatchTask(TaskPriority taskPriority) {
         TaskMetrics.incTaskDispatch();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskRecallProcessor.java
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index ae9a461d10..23577a90db 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -49,7 +49,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 /**
- * master scheduler thread
+ * Master scheduler thread, this thread will consume the commands from database and trigger processInstance executed.
  */
 @Service
 public class MasterSchedulerService extends Thread {
@@ -163,11 +163,8 @@ public class MasterSchedulerService extends Thread {
         MasterServerMetrics.incMasterConsumeCommand(commands.size());
 
         for (ProcessInstance processInstance : processInstances) {
-            if (processInstance == null) {
-                continue;
-            }
 
-            WorkflowExecuteRunnable workflowExecuteThread = new WorkflowExecuteRunnable(
+            WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
                     processInstance
                     , processService
                     , nettyExecutorManager
@@ -175,11 +172,11 @@ public class MasterSchedulerService extends Thread {
                     , masterConfig
                     , stateWheelExecuteThread);
 
-            this.processInstanceExecCacheManager.cache(processInstance.getId(), workflowExecuteThread);
+            this.processInstanceExecCacheManager.cache(processInstance.getId(), workflowExecuteRunnable);
             if (processInstance.getTimeout() > 0) {
                 stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
             }
-            workflowExecuteThreadPool.startWorkflow(workflowExecuteThread);
+            workflowExecuteThreadPool.startWorkflow(workflowExecuteRunnable);
         }
     }
 
@@ -203,7 +200,7 @@ public class MasterSchedulerService extends Thread {
                         logger.info("handle command {} end, create process instance {}", command.getId(), processInstance.getId());
                     }
                 } catch (Exception e) {
-                    logger.error("handle command error ", e);
+                    logger.error("handle command {} error ", command.getId(), e);
                     processService.moveToErrorCommand(command, e.toString());
                 } finally {
                     latch.countDown();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 7635209ebb..e85ddf08bd 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -31,8 +31,9 @@ import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheM
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
 import org.apache.dolphinscheduler.server.master.runner.task.TaskInstanceKey;
 
-import org.apache.hadoop.util.ThreadUtil;
+import org.apache.commons.lang3.ThreadUtils;
 
+import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentLinkedQueue;
 
@@ -84,6 +85,7 @@ public class StateWheelExecuteThread extends Thread {
 
     @Override
     public void run() {
+        Duration checkInterval = Duration.ofMillis(masterConfig.getStateWheelInterval() * Constants.SLEEP_TIME_MILLIS);
         while (Stopper.isRunning()) {
             try {
                 checkTask4Timeout();
@@ -93,7 +95,11 @@ public class StateWheelExecuteThread extends Thread {
             } catch (Exception e) {
                 logger.error("state wheel thread check error:", e);
             }
-            ThreadUtil.sleepAtLeastIgnoreInterrupts((long) masterConfig.getStateWheelInterval() * Constants.SLEEP_TIME_MILLIS);
+            try {
+                ThreadUtils.sleep(checkInterval);
+            } catch (InterruptedException e) {
+                logger.error("state wheel thread sleep error", e);
+            }
         }
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 82f942e03b..014abd30bc 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -805,11 +805,11 @@ public class WorkflowExecuteRunnable implements Runnable {
      */
     @Override
     public void run() {
-        if (this.taskInstanceMap.size() > 0) {
+        if (this.taskInstanceMap.size() > 0 || isStart) {
+            logger.warn("The workflow has already been started");
             return;
         }
         try {
-            isStart = false;
             buildFlowDag();
             initTaskQueue();
             submitPostNode(null);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index 36edf1371b..d8e2fbbeda 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -48,6 +48,9 @@ import org.springframework.util.concurrent.ListenableFutureCallback;
 
 import com.google.common.base.Strings;
 
+/**
+ * Used to execute {@link WorkflowExecuteRunnable}, when
+ */
 @Component
 public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
 
@@ -71,7 +74,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     /**
      * multi-thread filter, avoid handling workflow at the same time
      */
-    private ConcurrentHashMap<String, WorkflowExecuteRunnable> multiThreadFilterMap = new ConcurrentHashMap();
+    private ConcurrentHashMap<String, WorkflowExecuteRunnable> multiThreadFilterMap = new ConcurrentHashMap<>();
 
     @PostConstruct
     private void init() {
@@ -94,7 +97,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     }
 
     /**
-     * start workflow
+     * Start the given workflow.
      */
     public void startWorkflow(WorkflowExecuteRunnable workflowExecuteThread) {
         ProcessInstanceMetrics.incProcessInstanceSubmit();
@@ -102,13 +105,14 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     }
 
     /**
-     * execute workflow
+     * Handle the events belong to the given workflow.
      */
     public void executeEvent(WorkflowExecuteRunnable workflowExecuteThread) {
         if (!workflowExecuteThread.isStart() || workflowExecuteThread.eventSize() == 0) {
             return;
         }
         if (multiThreadFilterMap.containsKey(workflowExecuteThread.getKey())) {
+            logger.warn("The workflow:{} has been executed by another thread", workflowExecuteThread.getKey());
             return;
         }
         multiThreadFilterMap.put(workflowExecuteThread.getKey(), workflowExecuteThread);
@@ -123,8 +127,6 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
 
             @Override
             public void onSuccess(Object result) {
-                // if an exception occurs, first, the error message cannot be printed in the log;
-                // secondly, the `multiThreadFilterMap` cannot remove the `workflowExecuteThread`, resulting in the state of process instance cannot be changed and memory leak
                 try {
                     if (workflowExecuteThread.workFlowFinish()) {
                         stateWheelExecuteThread.removeProcess4TimeoutCheck(workflowExecuteThread.getProcessInstance());
@@ -134,8 +136,10 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
                     }
                 } catch (Exception e) {
                     logger.error("handle events {} success, but notify changed error", processInstanceId, e);
+                } finally {
+                    // make sure the process has been removed from multiThreadFilterMap
+                    multiThreadFilterMap.remove(workflowExecuteThread.getKey());
                 }
-                multiThreadFilterMap.remove(workflowExecuteThread.getKey());
             }
         });
     }
diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java
index 56033730bc..842cb86fd9 100644
--- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java
+++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java
@@ -303,6 +303,8 @@ public class NettyRemotingClient {
                 logger.error(msg, future.cause());
                 throw new RemotingException(msg);
             }
+        } catch (RemotingException remotingException) {
+            throw remotingException;
         } catch (Exception e) {
             logger.error("Send command {} to address {} encounter error.", command, host.getAddress());
             throw new RemotingException(String.format("Send command : %s , to :%s encounter error", command, host.getAddress()), e);
@@ -384,10 +386,10 @@ public class NettyRemotingClient {
                 if (this.responseFutureExecutor != null) {
                     this.responseFutureExecutor.shutdownNow();
                 }
+                logger.info("netty client closed");
             } catch (Exception ex) {
                 logger.error("netty client close exception", ex);
             }
-            logger.info("netty client closed");
         }
     }
 
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index a976c05c99..4bc9de32f2 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -17,7 +17,6 @@
 
 package org.apache.dolphinscheduler.service.process;
 
-import io.micrometer.core.annotation.Counted;
 import static java.util.stream.Collectors.toSet;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
@@ -161,6 +160,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
 
+import io.micrometer.core.annotation.Counted;
+
 /**
  * process relative dao that some mappers in this.
  */
@@ -1252,8 +1253,9 @@ public class ProcessServiceImpl implements ProcessService {
                 Thread.sleep(commitInterval);
             } catch (Exception e) {
                 logger.error("task commit to db failed", e);
+            } finally {
+                retryTimes += 1;
             }
-            retryTimes += 1;
         }
         return task;
     }
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java
index 8d630beeb0..fef5f8ff79 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/queue/TaskPriorityQueueImpl.java
@@ -29,15 +29,12 @@ import org.springframework.stereotype.Service;
  */
 @Service
 public class TaskPriorityQueueImpl implements TaskPriorityQueue<TaskPriority> {
-    /**
-     * queue size
-     */
-    private static final Integer QUEUE_MAX_SIZE = 3000;
 
     /**
-     * queue
+     * Task queue, this queue is unbounded, this means it will cause OutOfMemoryError.
+     * The master will stop to generate the task if memory is too high.
      */
-    private PriorityBlockingQueue<TaskPriority> queue = new PriorityBlockingQueue<>(QUEUE_MAX_SIZE);
+    private final PriorityBlockingQueue<TaskPriority> queue = new PriorityBlockingQueue<>(3000);
 
     /**
      * put task takePriorityInfo


[dolphinscheduler] 08/29: [Fix-10413] Fix Master startup failure the server still hang (#10500)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 90c87f012129e812d87a3f6e847de16f525611ea
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Mon Jun 20 22:35:06 2022 +0800

    [Fix-10413] Fix Master startup failure the server still hang (#10500)
    
    * Fix Master startup failure the server still hang
    
    (cherry picked from commit 117f78ec4b0e2438082a3e25158492eca1b9b1be)
---
 .../common/storage/StorageOperate.java             | 32 +++++++++----------
 .../common/thread/BaseDaemonThread.java            | 36 ++++++++++++++++++++++
 .../master/consumer/TaskPriorityQueueConsumer.java | 11 ++++---
 .../processor/queue/StateEventResponseService.java |  8 +++--
 .../master/processor/queue/TaskEventService.java   | 17 +++++++---
 .../server/master/runner/EventExecuteService.java  |  8 +++--
 .../master/runner/FailoverExecuteThread.java       | 10 ++++--
 .../master/runner/MasterSchedulerService.java      | 10 ++++--
 .../master/runner/StateWheelExecuteThread.java     |  7 ++++-
 9 files changed, 103 insertions(+), 36 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/storage/StorageOperate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/storage/StorageOperate.java
index 5248586de1..7854eaa032 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/storage/StorageOperate.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/storage/StorageOperate.java
@@ -28,21 +28,21 @@ import java.util.List;
 
 public interface StorageOperate {
 
-    public static final String RESOURCE_UPLOAD_PATH = PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
+    String RESOURCE_UPLOAD_PATH = PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler");
 
     /**
      * if the resource of tenant 's exist, the resource of folder will be created
      * @param tenantCode
      * @throws Exception
      */
-    public void createTenantDirIfNotExists(String tenantCode) throws Exception;
+    void createTenantDirIfNotExists(String tenantCode) throws Exception;
 
     /**
      * get the resource directory of tenant
      * @param tenantCode
      * @return
      */
-    public String getResDir(String tenantCode);
+    String getResDir(String tenantCode);
 
     /**
      * return the udf directory of tenant
@@ -50,7 +50,7 @@ public interface StorageOperate {
      * @return
      */
 
-    public String getUdfDir(String tenantCode);
+    String getUdfDir(String tenantCode);
 
     /**
      * create the directory that the path of tenant wanted to create
@@ -59,7 +59,7 @@ public interface StorageOperate {
      * @return
      * @throws IOException
      */
-    public boolean mkdir(String tenantCode,String path) throws IOException;
+    boolean mkdir(String tenantCode, String path) throws IOException;
 
     /**
      * get the path of the resource file
@@ -67,7 +67,7 @@ public interface StorageOperate {
      * @param fullName
      * @return
      */
-    public String getResourceFileName(String tenantCode, String fullName);
+    String getResourceFileName(String tenantCode, String fullName);
 
     /**
      * get the path of the file
@@ -76,7 +76,7 @@ public interface StorageOperate {
      * @param fileName
      * @return
      */
-    public String getFileName(ResourceType resourceType, String tenantCode, String fileName);
+    String getFileName(ResourceType resourceType, String tenantCode, String fileName);
 
     /**
      * predicate  if the resource of tenant exists
@@ -85,7 +85,7 @@ public interface StorageOperate {
      * @return
      * @throws IOException
      */
-    public  boolean exists(String tenantCode,String fileName) throws IOException;
+    boolean exists(String tenantCode, String fileName) throws IOException;
 
     /**
      * delete the resource of  filePath
@@ -96,7 +96,7 @@ public interface StorageOperate {
      * @return
      * @throws IOException
      */
-    public boolean delete(String tenantCode,String filePath, boolean recursive) throws IOException;
+    boolean delete(String tenantCode, String filePath, boolean recursive) throws IOException;
 
     /**
      * copy the file from srcPath to dstPath
@@ -107,7 +107,7 @@ public interface StorageOperate {
      * @return
      * @throws IOException
      */
-    public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException;
+    boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException;
 
     /**
      * get the root path of the tenant with resourceType
@@ -115,7 +115,7 @@ public interface StorageOperate {
      * @param tenantCode
      * @return
      */
-    public  String getDir(ResourceType resourceType, String tenantCode);
+    String getDir(ResourceType resourceType, String tenantCode);
 
     /**
      * upload the local srcFile to dstPath
@@ -127,7 +127,7 @@ public interface StorageOperate {
      * @return
      * @throws IOException
      */
-    public boolean upload(String tenantCode,String srcFile, String dstPath, boolean deleteSource, boolean overwrite) throws IOException;
+    boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, boolean overwrite) throws IOException;
 
     /**
      * download the srcPath to local
@@ -138,7 +138,7 @@ public interface StorageOperate {
      * @param overwrite
      * @throws IOException
      */
-    public void download(String tenantCode,String srcFilePath, String dstFile, boolean deleteSource, boolean overwrite)throws IOException;
+    void download(String tenantCode, String srcFilePath, String dstFile, boolean deleteSource, boolean overwrite)throws IOException;
 
     /**
      * vim the context of filePath
@@ -149,7 +149,7 @@ public interface StorageOperate {
      * @return
      * @throws IOException
      */
-    public List<String> vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException;
+    List<String> vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException;
 
     /**
      * delete the files and directory of the tenant
@@ -157,13 +157,13 @@ public interface StorageOperate {
      * @param tenantCode
      * @throws Exception
      */
-    public void deleteTenant(String tenantCode) throws Exception;
+    void deleteTenant(String tenantCode) throws Exception;
 
     /**
      * return the storageType
      *
      * @return
      */
-    public ResUploadType returnStorageType();
+    ResUploadType returnStorageType();
 
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/BaseDaemonThread.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/BaseDaemonThread.java
new file mode 100644
index 0000000000..88a44004cb
--- /dev/null
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/BaseDaemonThread.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.thread;
+
+/**
+ * All thread used in DolphinScheduler should extend with this class to avoid the server hang issue.
+ */
+public abstract class BaseDaemonThread extends Thread {
+
+    protected BaseDaemonThread(Runnable runnable) {
+        super(runnable);
+        this.setDaemon(true);
+    }
+
+    protected BaseDaemonThread(String threadName) {
+        super();
+        this.setName(threadName);
+        this.setDaemon(true);
+    }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
index d75595de1a..eab33a07a6 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/consumer/TaskPriorityQueueConsumer.java
@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.server.master.consumer;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@@ -60,7 +61,7 @@ import org.springframework.stereotype.Component;
  * TaskUpdateQueue consumer
  */
 @Component
-public class TaskPriorityQueueConsumer extends Thread {
+public class TaskPriorityQueueConsumer extends BaseDaemonThread {
 
     /**
      * logger of TaskUpdateQueueConsumer
@@ -108,6 +109,10 @@ public class TaskPriorityQueueConsumer extends Thread {
      */
     private ThreadPoolExecutor consumerThreadPoolExecutor;
 
+    protected TaskPriorityQueueConsumer() {
+        super("TaskPriorityQueueConsumeThread");
+    }
+
     @PostConstruct
     public void init() {
         this.consumerThreadPoolExecutor = (ThreadPoolExecutor) ThreadUtils.newDaemonFixedThreadExecutor("TaskUpdateQueueConsumerThread", masterConfig.getDispatchTaskNumber());
@@ -198,10 +203,8 @@ public class TaskPriorityQueueConsumer extends Thread {
             } else {
                 logger.info("Master failed to dispatch task to worker, taskInstanceId: {}", taskPriority.getTaskId());
             }
-        } catch (RuntimeException e) {
+        } catch (RuntimeException | ExecuteException e) {
             logger.error("Master dispatch task to worker error: ", e);
-        } catch (ExecuteException e) {
-            logger.error("Master dispatch task to worker error: {}", e);
         }
         return result;
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
index 83772a1054..e34dedca67 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
 import org.apache.dolphinscheduler.common.enums.StateEvent;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.StateEventResponseCommand;
@@ -70,7 +71,6 @@ public class StateEventResponseService {
     @PostConstruct
     public void start() {
         this.responseWorker = new StateEventResponseWorker();
-        this.responseWorker.setName("StateEventResponseWorker");
         this.responseWorker.start();
     }
 
@@ -101,7 +101,11 @@ public class StateEventResponseService {
     /**
      * task worker thread
      */
-    class StateEventResponseWorker extends Thread {
+    class StateEventResponseWorker extends BaseDaemonThread {
+
+        protected StateEventResponseWorker() {
+            super("StateEventResponseWorker");
+        }
 
         @Override
         public void run() {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java
index 924dd131f8..bed3b3d9ed 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskEventService.java
@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.server.master.processor.queue;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 
 import java.util.ArrayList;
@@ -63,11 +64,9 @@ public class TaskEventService {
     @PostConstruct
     public void start() {
         this.taskEventThread = new TaskEventThread();
-        this.taskEventThread.setName("TaskEventThread");
         this.taskEventThread.start();
 
         this.taskEventHandlerThread = new TaskEventHandlerThread();
-        this.taskEventHandlerThread.setName("TaskEventHandlerThread");
         this.taskEventHandlerThread.start();
     }
 
@@ -85,7 +84,7 @@ public class TaskEventService {
                 taskExecuteThreadPool.eventHandler();
             }
         } catch (Exception e) {
-            logger.error("stop error:", e);
+            logger.error("TaskEventService stop error:", e);
         }
     }
 
@@ -101,7 +100,11 @@ public class TaskEventService {
     /**
      * task worker thread
      */
-    class TaskEventThread extends Thread {
+    class TaskEventThread extends BaseDaemonThread {
+        protected TaskEventThread() {
+            super("TaskEventLoopThread");
+        }
+
         @Override
         public void run() {
             while (Stopper.isRunning()) {
@@ -123,7 +126,11 @@ public class TaskEventService {
     /**
      * event handler thread
      */
-    class TaskEventHandlerThread extends Thread {
+    class TaskEventHandlerThread extends BaseDaemonThread {
+
+        protected TaskEventHandlerThread() {
+            super("TaskEventHandlerThread");
+        }
 
         @Override
         public void run() {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
index fc9c4fd854..97c67d8493 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.server.master.runner;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 
@@ -29,7 +30,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 @Service
-public class EventExecuteService extends Thread {
+public class EventExecuteService extends BaseDaemonThread {
 
     private static final Logger logger = LoggerFactory.getLogger(EventExecuteService.class);
 
@@ -42,9 +43,12 @@ public class EventExecuteService extends Thread {
     @Autowired
     private WorkflowExecuteThreadPool workflowExecuteThreadPool;
 
+    protected EventExecuteService() {
+        super("EventServiceStarted");
+    }
+
     @Override
     public synchronized void start() {
-        super.setName("EventServiceStarted");
         super.start();
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
index 4bac6cdf15..1c6ea144e4 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.server.master.runner;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.server.master.config.MasterConfig;
@@ -29,7 +30,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 @Service
-public class FailoverExecuteThread extends Thread {
+public class FailoverExecuteThread extends BaseDaemonThread {
 
     private static final Logger logger = LoggerFactory.getLogger(FailoverExecuteThread.class);
 
@@ -42,16 +43,19 @@ public class FailoverExecuteThread extends Thread {
     @Autowired
     private FailoverService failoverService;
 
+    protected FailoverExecuteThread() {
+        super("FailoverExecuteThread");
+    }
+
     @Override
     public synchronized void start() {
-        super.setName("FailoverExecuteThread");
         super.start();
     }
 
     @Override
     public void run() {
         // when startup, wait 10s for ready
-        ThreadUtils.sleep((long) Constants.SLEEP_TIME_MILLIS * 10);
+        ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * 10);
 
         logger.info("failover execute thread started");
         while (Stopper.isRunning()) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index 23577a90db..94991a50c3 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.server.master.runner;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.SlotCheckState;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
@@ -52,7 +53,7 @@ import org.springframework.stereotype.Service;
  * Master scheduler thread, this thread will consume the commands from database and trigger processInstance executed.
  */
 @Service
-public class MasterSchedulerService extends Thread {
+public class MasterSchedulerService extends BaseDaemonThread {
 
     /**
      * logger of MasterSchedulerService
@@ -102,6 +103,10 @@ public class MasterSchedulerService extends Thread {
     @Autowired
     private StateWheelExecuteThread stateWheelExecuteThread;
 
+    protected MasterSchedulerService() {
+        super("MasterCommandLoopThread");
+    }
+
     /**
      * constructor of MasterSchedulerService
      */
@@ -113,9 +118,8 @@ public class MasterSchedulerService extends Thread {
 
     @Override
     public synchronized void start() {
-        super.setName("MasterSchedulerService");
-        super.start();
         this.stateWheelExecuteThread.start();
+        super.start();
     }
 
     public void close() {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index e85ddf08bd..12d404406c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
@@ -50,7 +51,7 @@ import org.springframework.stereotype.Component;
  * 4. timeout process check
  */
 @Component
-public class StateWheelExecuteThread extends Thread {
+public class StateWheelExecuteThread extends BaseDaemonThread {
 
     private static final Logger logger = LoggerFactory.getLogger(StateWheelExecuteThread.class);
 
@@ -83,6 +84,10 @@ public class StateWheelExecuteThread extends Thread {
     @Autowired
     private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
 
+    protected StateWheelExecuteThread() {
+        super("StateWheelExecuteThread");
+    }
+
     @Override
     public void run() {
         Duration checkInterval = Duration.ofMillis(masterConfig.getStateWheelInterval() * Constants.SLEEP_TIME_MILLIS);


[dolphinscheduler] 11/29: add CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit fc1c1f6ad1d3f3d1cfb32afc59c0d5b2c94f0770
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jul 19 11:30:47 2022 +0800

    add CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST
---
 .../main/java/org/apache/dolphinscheduler/common/Constants.java    | 5 +++++
 .../server/master/runner/WorkflowExecuteRunnable.java              | 7 +++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index f39a0c94aa..5b813b22c7 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -362,6 +362,11 @@ public final class Constants {
      */
     public static final String CMDPARAM_COMPLEMENT_DATA_END_DATE = "complementEndDate";
 
+    /**
+     * complement data Schedule date
+     */
+    public static final String CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST = "complementScheduleDateList";
+
     /**
      * complement date default cron string
      */
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 9c5f4062f7..ab74f6e227 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -18,11 +18,14 @@
 package org.apache.dolphinscheduler.server.master.runner;
 
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
+import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST;
 import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
 import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
+import static org.apache.dolphinscheduler.common.Constants.COMMA;
 import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
+import static org.apache.dolphinscheduler.common.Constants.YYYY_MM_DD_HH_MM_SS;
 import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
 import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR;
 import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
@@ -1930,7 +1933,7 @@ public class WorkflowExecuteRunnable implements Runnable {
         Map<String, String> paramMap = JSONUtils.toMap(cmdParam);
 
         if (paramMap != null && paramMap.containsKey(CMD_PARAM_RECOVERY_START_NODE_STRING)) {
-            String[] idList = paramMap.get(CMD_PARAM_RECOVERY_START_NODE_STRING).split(Constants.COMMA);
+            String[] idList = paramMap.get(CMD_PARAM_RECOVERY_START_NODE_STRING).split(COMMA);
             instanceList = getRecoverTaskInstanceList(idList);
         }
         return instanceList;
@@ -1949,7 +1952,7 @@ public class WorkflowExecuteRunnable implements Runnable {
             return startNodeNameList;
         }
         if (paramMap.containsKey(CMD_PARAM_START_NODES)) {
-            startNodeNameList = Arrays.asList(paramMap.get(CMD_PARAM_START_NODES).split(Constants.COMMA));
+            startNodeNameList = Arrays.asList(paramMap.get(CMD_PARAM_START_NODES).split(COMMA));
         }
         return startNodeNameList;
     }


[dolphinscheduler] 18/29: Add task prepare metrics

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 04c47034d43d7795e13c1dcf156e1f6bcf31e65f
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Jul 19 11:52:18 2022 +0800

    Add task prepare metrics
---
 .../server/master/metrics/TaskMetrics.java         | 28 +++++++++++-----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java
index c4516d0644..3f2ed1544f 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/metrics/TaskMetrics.java
@@ -30,57 +30,57 @@ public final class TaskMetrics {
     }
 
     private static final Counter TASK_SUBMIT_COUNTER =
-            Counter.builder("dolphinscheduler_task_submit_count")
+            Counter.builder("ds.task.submit.count")
                     .description("Task submit total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_FINISH_COUNTER =
-            Counter.builder("dolphinscheduler_task_finish_count")
+            Counter.builder("ds.task.finish.count")
                     .description("Task finish total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_SUCCESS_COUNTER =
-            Counter.builder("dolphinscheduler_task_success_count")
+            Counter.builder("ds.task.success.count")
                     .description("Task success total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_FAILURE_COUNTER =
-            Counter.builder("dolphinscheduler_task_failure_count")
+            Counter.builder("ds.task.failure.count")
                     .description("Task failure total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_TIMEOUT_COUNTER =
-            Counter.builder("dolphinscheduler_task_timeout_count")
+            Counter.builder("ds.task.timeout.count")
                     .description("Task timeout total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_RETRY_COUNTER =
-            Counter.builder("dolphinscheduler_task_retry_count")
+            Counter.builder("ds.task.retry.count")
                     .description("Task retry total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_STOP_COUNTER =
-            Counter.builder("dolphinscheduler_task_stop_count")
+            Counter.builder("ds.task.stop.count")
                     .description("Task stop total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_FAILOVER_COUNTER =
-            Counter.builder("dolphinscheduler_task_failover_count")
+            Counter.builder("ds.task.failover.count")
                     .description("Task failover total count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_DISPATCH_COUNTER =
-            Counter.builder("dolphinscheduler_task_dispatch_count")
+            Counter.builder("ds.task.dispatch.count")
                     .description("Task dispatch count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_DISPATCHER_FAILED =
-            Counter.builder("dolphinscheduler_task_dispatch_failed_count")
+            Counter.builder("ds.task.dispatch.failure.count")
                     .description("Task dispatch failed count")
                     .register(Metrics.globalRegistry);
 
     private static final Counter TASK_DISPATCH_ERROR =
-            Counter.builder("dolphinscheduler_task_dispatch_error_count")
+            Counter.builder("ds.task.dispatch.error.count")
                     .description("Task dispatch error")
                     .register(Metrics.globalRegistry);
 
@@ -88,9 +88,9 @@ public final class TaskMetrics {
         TASK_SUBMIT_COUNTER.increment();
     }
 
-    public synchronized static void registerTaskRunning(Supplier<Number> consumer) {
-        Gauge.builder("dolphinscheduler_task_running_gauge", consumer)
-                .description("Task running count")
+    public synchronized static void registerTaskPrepared(Supplier<Number> consumer) {
+        Gauge.builder("ds.task.prepared", consumer)
+                .description("Task prepared count")
                 .register(Metrics.globalRegistry);
     }
 


[dolphinscheduler] 10/29: Optimize master log, use MDC to inject workflow instance id and task instance id in log (#10516)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 3ab9ee13fcffcfe69de90353ad43f8eb86c015a6
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Thu Jun 23 11:45:06 2022 +0800

    Optimize master log, use MDC to inject workflow instance id and task instance id in log (#10516)
    
    * Optimize master log, add workflow instance id and task instance id in log
    
    * Use MDC to set the workflow info in log4j
    
    * Add workflowInstanceId and taskInstanceId in MDC
    
    (cherry picked from commit db595b3eff8f5e4728665b4b7b7082af710e8f9d)
---
 .../apache/dolphinscheduler/alert/AlertServer.java |  23 ++-
 .../apache/dolphinscheduler/common/Constants.java  |  36 ++--
 .../dolphinscheduler/common/thread/Stopper.java    |  34 +++-
 .../common/thread/ThreadUtils.java                 |  18 +-
 .../dolphinscheduler/common/utils/LoggerUtils.java |  34 +++-
 .../server/master/MasterServer.java                |  25 +--
 .../master/processor/StateEventProcessor.java      |  12 +-
 .../master/processor/TaskEventProcessor.java       |  14 +-
 .../processor/TaskExecuteResponseProcessor.java    |  12 +-
 .../processor/TaskKillResponseProcessor.java       |   3 +-
 .../processor/queue/StateEventResponseService.java |  23 ++-
 .../processor/queue/TaskExecuteRunnable.java       |   3 +
 .../processor/queue/TaskExecuteThreadPool.java     |  12 +-
 .../master/registry/MasterRegistryClient.java      |  15 +-
 .../server/master/runner/EventExecuteService.java  |  18 +-
 .../master/runner/FailoverExecuteThread.java       |   5 +-
 .../master/runner/MasterSchedulerService.java      |  60 ++++--
 .../master/runner/StateWheelExecuteThread.java     | 222 +++++++++++----------
 .../master/runner/WorkflowExecuteRunnable.java     |  24 ++-
 .../master/runner/WorkflowExecuteThreadPool.java   |  29 ++-
 .../master/runner/task/TaskProcessorFactory.java   |   9 +-
 .../server/master/service/FailoverService.java     |  24 ++-
 .../src/main/resources/logback-spring.xml          |   4 +-
 .../service/alert/AlertClientService.java          |  21 +-
 .../service/task/TaskPluginManager.java            |   5 +-
 .../src/main/resources/logback-spring.xml          |   4 +-
 .../plugin/task/api/ProcessUtils.java              |  15 +-
 .../plugin/task/api/TaskPluginException.java       |  23 +--
 .../server/worker/WorkerServer.java                | 117 +++--------
 .../server/worker/prc/WorkerRpcServer.java         |  97 +++++++++
 .../worker/processor/TaskExecuteProcessor.java     | 169 +++++++++-------
 .../processor/TaskExecuteRunningAckProcessor.java  |  17 +-
 .../worker/runner/RetryReportTaskStatusThread.java |   4 +-
 .../server/worker/runner/TaskExecuteThread.java    |  31 ++-
 .../server/worker/runner/WorkerManagerThread.java  |   5 +-
 .../src/main/resources/logback-spring.xml          |   4 +-
 .../worker/processor/TaskExecuteProcessorTest.java |  42 ++--
 .../worker/runner/TaskExecuteThreadTest.java       |  27 ++-
 38 files changed, 747 insertions(+), 493 deletions(-)

diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
index ee9d5b3f62..5f449fcab4 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
@@ -49,7 +49,10 @@ public class AlertServer implements Closeable {
     private final AlertConfig alertConfig;
     private NettyRemotingServer nettyRemotingServer;
 
-    public AlertServer(PluginDao pluginDao, AlertSenderService alertSenderService, AlertRequestProcessor alertRequestProcessor, AlertConfig alertConfig) {
+    public AlertServer(PluginDao pluginDao,
+                       AlertSenderService alertSenderService,
+                       AlertRequestProcessor alertRequestProcessor,
+                       AlertConfig alertConfig) {
         this.pluginDao = pluginDao;
         this.alertSenderService = alertSenderService;
         this.alertRequestProcessor = alertRequestProcessor;
@@ -68,11 +71,12 @@ public class AlertServer implements Closeable {
 
     @EventListener
     public void run(ApplicationReadyEvent readyEvent) {
-        logger.info("alert server starting...");
+        logger.info("Alert server is staring ...");
 
         checkTable();
         startServer();
         alertSenderService.start();
+        logger.info("Alert server is started ...");
     }
 
     @Override
@@ -89,24 +93,23 @@ public class AlertServer implements Closeable {
     public void destroy(String cause) {
 
         try {
+            // set stop signal is true
             // execute only once
-            if (Stopper.isStopped()) {
+            if (!Stopper.stop()) {
+                logger.warn("AlterServer is already stopped");
                 return;
             }
 
-            logger.info("alert server is stopping ..., cause : {}", cause);
-
-            // set stop signal is true
-            Stopper.stop();
+            logger.info("Alert server is stopping, cause: {}", cause);
 
             // thread sleep 3 seconds for thread quietly stop
-            ThreadUtils.sleep(3000L);
+            ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
 
             // close
             this.nettyRemotingServer.close();
-
+            logger.info("Alter server stopped, cause: {}", cause);
         } catch (Exception e) {
-            logger.error("alert server stop exception ", e);
+            logger.error("Alert server stop failed, cause: {}", cause, e);
         }
     }
 
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index b7714c7817..f39a0c94aa 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.SystemUtils;
 
+import java.time.Duration;
 import java.util.regex.Pattern;
 
 /**
@@ -376,6 +377,8 @@ public final class Constants {
      */
     public static final long SLEEP_TIME_MILLIS_SHORT = 100L;
 
+    public static final Duration SERVER_CLOSE_WAIT_TIME = Duration.ofSeconds(3);
+
     /**
      * one second mils
      */
@@ -636,28 +639,31 @@ public final class Constants {
      */
     public static final String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path";
 
+    public static final String WORKFLOW_INSTANCE_ID_MDC_KEY = "workflowInstanceId";
+    public static final String TASK_INSTANCE_ID_MDC_KEY = "taskInstanceId";
+
     /**
      * task log info format
      */
     public static final String TASK_LOG_INFO_FORMAT = "TaskLogInfo-%s";
 
-    public static final int[] NOT_TERMINATED_STATES = new int[]{
-            ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
-            ExecutionStatus.DISPATCH.ordinal(),
-            ExecutionStatus.RUNNING_EXECUTION.ordinal(),
-            ExecutionStatus.DELAY_EXECUTION.ordinal(),
-            ExecutionStatus.READY_PAUSE.ordinal(),
-            ExecutionStatus.READY_STOP.ordinal(),
-            ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(),
-            ExecutionStatus.WAITING_THREAD.ordinal(),
-            ExecutionStatus.WAITING_DEPEND.ordinal()
+    public static final int[] NOT_TERMINATED_STATES = new int[] {
+        ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
+        ExecutionStatus.DISPATCH.ordinal(),
+        ExecutionStatus.RUNNING_EXECUTION.ordinal(),
+        ExecutionStatus.DELAY_EXECUTION.ordinal(),
+        ExecutionStatus.READY_PAUSE.ordinal(),
+        ExecutionStatus.READY_STOP.ordinal(),
+        ExecutionStatus.NEED_FAULT_TOLERANCE.ordinal(),
+        ExecutionStatus.WAITING_THREAD.ordinal(),
+        ExecutionStatus.WAITING_DEPEND.ordinal()
     };
 
-    public static final int[] RUNNING_PROCESS_STATE = new int[]{
-            ExecutionStatus.RUNNING_EXECUTION.ordinal(),
-            ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
-            ExecutionStatus.DISPATCH.ordinal(),
-            ExecutionStatus.SERIAL_WAIT.ordinal()
+    public static final int[] RUNNING_PROCESS_STATE = new int[] {
+        ExecutionStatus.RUNNING_EXECUTION.ordinal(),
+        ExecutionStatus.SUBMITTED_SUCCESS.ordinal(),
+        ExecutionStatus.DISPATCH.ordinal(),
+        ExecutionStatus.SERIAL_WAIT.ordinal()
     };
 
     /**
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java
index 67abde7e7a..777203c0f3 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java
@@ -19,22 +19,40 @@ package org.apache.dolphinscheduler.common.thread;
 
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import lombok.experimental.UtilityClass;
+
 /**
- *  if the process closes, a signal is placed as true, and all threads get this flag to stop working
+ * If the process closes, a signal is placed as true, and all threads get this flag to stop working.
  */
+@UtilityClass
 public class Stopper {
 
-    private static AtomicBoolean signal = new AtomicBoolean(false);
+    private static final AtomicBoolean stoppedSignal = new AtomicBoolean(false);
 
-    public static final boolean isStopped() {
-        return signal.get();
+    /**
+     * Return the flag if the Server is stopped.
+     *
+     * @return True, if the server is stopped; False, the server is still running.
+     */
+    public static boolean isStopped() {
+        return stoppedSignal.get();
     }
 
-    public static final boolean isRunning() {
-        return !signal.get();
+    /**
+     * Return the flag if the Server is stopped.
+     *
+     * @return True, if the server is running, False, the server is stopped.
+     */
+    public static boolean isRunning() {
+        return !stoppedSignal.get();
     }
 
-    public static final void stop() {
-        signal.set(true);
+    /**
+     * Stop the server
+     *
+     * @return True, if the server stopped success. False, if the server is already stopped.
+     */
+    public static boolean stop() {
+        return stoppedSignal.compareAndSet(false, true);
     }
 }
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
index 75f624d15d..5c8020b7cd 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
@@ -23,24 +23,28 @@ import java.util.concurrent.ThreadFactory;
 
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
-/**
- * thread utils
- */
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class ThreadUtils {
     /**
      * Wrapper over newDaemonFixedThreadExecutor.
+     *
      * @param threadName threadName
      * @param threadsNum threadsNum
      * @return ExecutorService
      */
-    public static ExecutorService newDaemonFixedThreadExecutor(String threadName,int threadsNum) {
+    public static ExecutorService newDaemonFixedThreadExecutor(String threadName, int threadsNum) {
         ThreadFactory threadFactory = new ThreadFactoryBuilder()
-                .setDaemon(true)
-                .setNameFormat(threadName)
-                .build();
+            .setDaemon(true)
+            .setNameFormat(threadName)
+            .build();
         return Executors.newFixedThreadPool(threadsNum, threadFactory);
     }
 
+    /**
+     * Sleep in given mills, this is not accuracy.
+     */
     public static void sleep(final long millis) {
         try {
             Thread.sleep(millis);
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
index 8dd4969494..8c749c53fc 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
@@ -32,16 +32,16 @@ import java.util.regex.Pattern;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import lombok.experimental.UtilityClass;
 
 /**
  * logger utils
  */
+@UtilityClass
 public class LoggerUtils {
 
-    private LoggerUtils() {
-        throw new UnsupportedOperationException("Construct LoggerUtils");
-    }
-
     private static final Logger logger = LoggerFactory.getLogger(LoggerUtils.class);
 
     /**
@@ -109,4 +109,30 @@ public class LoggerUtils {
         }
         return "";
     }
+
+    public static void setWorkflowAndTaskInstanceIDMDC(int workflowInstanceId, int taskInstanceId) {
+        setWorkflowInstanceIdMDC(workflowInstanceId);
+        setTaskInstanceIdMDC(taskInstanceId);
+    }
+
+    public static void setWorkflowInstanceIdMDC(int workflowInstanceId) {
+        MDC.put(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY, String.valueOf(workflowInstanceId));
+    }
+
+    public static void setTaskInstanceIdMDC(int taskInstanceId) {
+        MDC.put(Constants.TASK_INSTANCE_ID_MDC_KEY, String.valueOf(taskInstanceId));
+    }
+
+    public static void removeWorkflowAndTaskInstanceIdMDC() {
+        removeWorkflowInstanceIdMDC();
+        removeTaskInstanceIdMDC();
+    }
+
+    public static void removeWorkflowInstanceIdMDC() {
+        MDC.remove(Constants.WORKFLOW_INSTANCE_ID_MDC_KEY);
+    }
+
+    public static void removeTaskInstanceIdMDC() {
+        MDC.remove(Constants.TASK_INSTANCE_ID_MDC_KEY);
+    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index cb01e14504..95409a616b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.IStoppable;
 import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
 import org.apache.dolphinscheduler.server.master.rpc.MasterRPCServer;
 import org.apache.dolphinscheduler.server.master.runner.EventExecuteService;
@@ -103,7 +104,7 @@ public class MasterServer implements IStoppable {
 
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             if (Stopper.isRunning()) {
-                close("shutdownHook");
+                close("MasterServer shutdownHook");
             }
         }));
     }
@@ -116,23 +117,17 @@ public class MasterServer implements IStoppable {
     public void close(String cause) {
 
         try {
+            // set stop signal is true
             // execute only once
-            if (Stopper.isStopped()) {
-                logger.warn("MasterServer has been stopped ..., current cause: {}", cause);
+            if (!Stopper.stop()) {
+                logger.warn("MasterServer is already stopped, current cause: {}", cause);
                 return;
             }
 
-            logger.info("master server is stopping ..., cause : {}", cause);
-
-            // set stop signal is true
-            Stopper.stop();
+            logger.info("Master server is stopping, current cause : {}", cause);
 
-            try {
-                // thread sleep 3 seconds for thread quietly stop
-                Thread.sleep(3000L);
-            } catch (Exception e) {
-                logger.warn("thread sleep exception ", e);
-            }
+            // thread sleep 3 seconds for thread quietly stop
+            ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
             // close
             this.masterSchedulerService.close();
             this.masterRPCServer.close();
@@ -141,9 +136,9 @@ public class MasterServer implements IStoppable {
             // like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc
             springApplicationContext.close();
 
-            logger.info("MasterServer stopped...");
+            logger.info("MasterServer stopped, current cause: {}", cause);
         } catch (Exception e) {
-            logger.error("master server stop exception ", e);
+            logger.error("MasterServer stop failed, current cause: {}", cause, e);
         }
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
index e132f285de..b2c47e4112 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/StateEventProcessor.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.processor;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -64,8 +65,15 @@ public class StateEventProcessor implements NettyRequestProcessor {
         StateEventType type = stateEvent.getTaskInstanceId() == 0 ? StateEventType.PROCESS_STATE_CHANGE : StateEventType.TASK_STATE_CHANGE;
         stateEvent.setType(type);
 
-        logger.info("received command : {}", stateEvent);
-        stateEventResponseService.addResponse(stateEvent);
+        try {
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
+
+            logger.info("Received state event change command, event: {}", stateEvent);
+            stateEventResponseService.addResponse(stateEvent);
+        }finally {
+            LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
+        }
+
     }
 
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
index ea2e4539c3..e597f1cae5 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskEventProcessor.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.processor;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.command.TaskEventChangeCommand;
@@ -49,8 +50,8 @@ public class TaskEventProcessor implements NettyRequestProcessor {
     @Override
     public void process(Channel channel, Command command) {
         Preconditions.checkArgument(CommandType.TASK_FORCE_STATE_EVENT_REQUEST == command.getType()
-                        || CommandType.TASK_WAKEUP_EVENT_REQUEST == command.getType()
-                , String.format("invalid command type: %s", command.getType()));
+                || CommandType.TASK_WAKEUP_EVENT_REQUEST == command.getType()
+            , String.format("invalid command type: %s", command.getType()));
 
         TaskEventChangeCommand taskEventChangeCommand = JSONUtils.parseObject(command.getBody(), TaskEventChangeCommand.class);
         StateEvent stateEvent = new StateEvent();
@@ -58,8 +59,13 @@ public class TaskEventProcessor implements NettyRequestProcessor {
         stateEvent.setProcessInstanceId(taskEventChangeCommand.getProcessInstanceId());
         stateEvent.setTaskInstanceId(taskEventChangeCommand.getTaskInstanceId());
         stateEvent.setType(StateEventType.WAIT_TASK_GROUP);
-        logger.info("received command : {}", stateEvent);
-        stateEventResponseService.addEvent2WorkflowExecute(stateEvent);
+        try {
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
+            logger.info("Received task event change command, event: {}", stateEvent);
+            stateEventResponseService.addEvent2WorkflowExecute(stateEvent);
+        } finally {
+            LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
+        }
     }
 
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
index 264e42e3a5..b17def873c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskExecuteResponseProcessor.java
@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.server.master.processor;
 
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
@@ -57,9 +58,14 @@ public class TaskExecuteResponseProcessor implements NettyRequestProcessor {
         Preconditions.checkArgument(CommandType.TASK_EXECUTE_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType()));
 
         TaskExecuteResponseCommand taskExecuteResponseCommand = JSONUtils.parseObject(command.getBody(), TaskExecuteResponseCommand.class);
-        logger.info("received command : {}", taskExecuteResponseCommand);
-
         TaskEvent taskResponseEvent = TaskEvent.newResultEvent(taskExecuteResponseCommand, channel);
-        taskEventService.addEvent(taskResponseEvent);
+        try {
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskResponseEvent.getProcessInstanceId(), taskResponseEvent.getTaskInstanceId());
+            logger.info("Received task execute response, event: {}", taskResponseEvent);
+
+            taskEventService.addEvent(taskResponseEvent);
+        } finally {
+            LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
+        }
     }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java
index 135257c9a8..6079329b90 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/TaskKillResponseProcessor.java
@@ -51,7 +51,8 @@ public class TaskKillResponseProcessor implements NettyRequestProcessor {
         Preconditions.checkArgument(CommandType.TASK_KILL_RESPONSE == command.getType(), String.format("invalid command type : %s", command.getType()));
 
         TaskKillResponseCommand responseCommand = JSONUtils.parseObject(command.getBody(), TaskKillResponseCommand.class);
-        logger.info("received task kill response command : {}", responseCommand);
+        logger.info("[TaskInstance-{}] Received task kill response command : {}",
+            responseCommand.getTaskInstanceId(), responseCommand);
     }
 
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
index e34dedca67..4ca6b9eccb 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/StateEventResponseService.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.processor.queue;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.StateEventResponseCommand;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
@@ -81,7 +82,13 @@ public class StateEventResponseService {
             List<StateEvent> remainEvents = new ArrayList<>(eventQueue.size());
             eventQueue.drainTo(remainEvents);
             for (StateEvent event : remainEvents) {
-                this.persist(event);
+                try {
+                    LoggerUtils.setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), event.getTaskInstanceId());
+                    this.persist(event);
+
+                } finally {
+                    LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
+                }
             }
         }
     }
@@ -93,7 +100,7 @@ public class StateEventResponseService {
         try {
             eventQueue.put(stateEvent);
         } catch (InterruptedException e) {
-            logger.error("put state event : {} error :{}", stateEvent, e);
+            logger.error("Put state event : {} error", stateEvent, e);
             Thread.currentThread().interrupt();
         }
     }
@@ -109,18 +116,22 @@ public class StateEventResponseService {
 
         @Override
         public void run() {
-
+            logger.info("State event loop service started");
             while (Stopper.isRunning()) {
                 try {
                     // if not task , blocking here
                     StateEvent stateEvent = eventQueue.take();
+                    LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
                     persist(stateEvent);
                 } catch (InterruptedException e) {
+                    logger.warn("State event loop service interrupted, will stop this loop", e);
                     Thread.currentThread().interrupt();
                     break;
+                } finally {
+                    LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
                 }
             }
-            logger.info("StateEventResponseWorker stopped");
+            logger.info("State event loop service stopped");
         }
     }
 
@@ -135,6 +146,8 @@ public class StateEventResponseService {
     private void persist(StateEvent stateEvent) {
         try {
             if (!this.processInstanceExecCacheManager.contains(stateEvent.getProcessInstanceId())) {
+                logger.warn("Persist event into workflow execute thread error, "
+                    + "cannot find the workflow instance from cache manager, event: {}", stateEvent);
                 writeResponse(stateEvent, ExecutionStatus.FAILURE);
                 return;
             }
@@ -152,7 +165,7 @@ public class StateEventResponseService {
             workflowExecuteThreadPool.submitStateEvent(stateEvent);
             writeResponse(stateEvent, ExecutionStatus.SUCCESS);
         } catch (Exception e) {
-            logger.error("persist event queue error, event: {}", stateEvent, e);
+            logger.error("Persist event queue error, event: {}", stateEvent, e);
         }
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
index 593567c6eb..3bf249ed3e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteRunnable.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.processor.queue;
 import org.apache.dolphinscheduler.common.enums.Event;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseAckCommand;
@@ -71,11 +72,13 @@ public class TaskExecuteRunnable implements Runnable {
         while (!this.events.isEmpty()) {
             TaskEvent event = this.events.peek();
             try {
+                LoggerUtils.setWorkflowAndTaskInstanceIDMDC(event.getProcessInstanceId(), event.getTaskInstanceId());
                 persist(event);
             } catch (Exception e) {
                 logger.error("persist error, event:{}, error: {}", event, e);
             } finally {
                 this.events.remove(event);
+                LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
             }
         }
     }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
index 323ea86411..68e4511c42 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/processor/queue/TaskExecuteThreadPool.java
@@ -115,11 +115,11 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
             @Override
             public void onFailure(Throwable ex) {
                 Integer processInstanceId = taskExecuteThread.getProcessInstanceId();
-                logger.error("persist event failed processInstanceId: {}", processInstanceId, ex);
+                logger.error("[WorkflowInstance-{}] persist event failed", processInstanceId, ex);
                 if (!processInstanceExecCacheManager.contains(processInstanceId)) {
                     taskExecuteThreadMap.remove(processInstanceId);
-                    logger.info("Cannot find processInstance from cacheManager, remove process instance from threadMap: {}",
-                            processInstanceId);
+                    logger.info("[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap",
+                        processInstanceId);
                 }
                 multiThreadFilterMap.remove(taskExecuteThread.getKey());
             }
@@ -127,11 +127,11 @@ public class TaskExecuteThreadPool extends ThreadPoolTaskExecutor {
             @Override
             public void onSuccess(Object result) {
                 Integer processInstanceId = taskExecuteThread.getProcessInstanceId();
-                logger.info("persist events succeeded, processInstanceId: {}", processInstanceId);
+                logger.info("[WorkflowInstance-{}] persist events succeeded", processInstanceId);
                 if (!processInstanceExecCacheManager.contains(processInstanceId)) {
                     taskExecuteThreadMap.remove(processInstanceId);
-                    logger.info("Cannot find processInstance from cacheManager, remove process instance from threadMap: {}",
-                            processInstanceId);
+                    logger.info("[WorkflowInstance-{}] Cannot find processInstance from cacheManager, remove process instance from threadMap",
+                        processInstanceId);
                 }
                 multiThreadFilterMap.remove(taskExecuteThread.getKey());
             }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
index 5cf1c945af..771bac4a7a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
@@ -99,7 +99,6 @@ public class MasterRegistryClient {
                                                                                         registryClient));
             registryClient.subscribe(REGISTRY_DOLPHINSCHEDULER_NODE, new MasterRegistryDataListener());
         } catch (Exception e) {
-            logger.error("master start up exception", e);
             throw new RegistryException("Master registry client start up error", e);
         }
     }
@@ -186,7 +185,7 @@ public class MasterRegistryClient {
      * Registry the current master server itself to registry.
      */
     void registry() {
-        logger.info("master node : {} registering to registry center...", masterAddress);
+        logger.info("Master node : {} registering to registry center", masterAddress);
         String localNodePath = getCurrentNodePath();
         int masterHeartbeatInterval = masterConfig.getHeartbeatInterval();
         HeartBeatTask heartBeatTask = new HeartBeatTask(startupTime,
@@ -201,7 +200,7 @@ public class MasterRegistryClient {
         registryClient.persistEphemeral(localNodePath, heartBeatTask.getHeartBeatInfo());
 
         while (!registryClient.checkNodeExists(NetUtils.getHost(), NodeType.MASTER)) {
-            logger.warn("The current master server node:{} cannot find in registry....", NetUtils.getHost());
+            logger.warn("The current master server node:{} cannot find in registry", NetUtils.getHost());
             ThreadUtils.sleep(SLEEP_TIME_MILLIS);
         }
 
@@ -212,9 +211,7 @@ public class MasterRegistryClient {
         registryClient.handleDeadServer(Collections.singleton(localNodePath), NodeType.MASTER, Constants.DELETE_OP);
 
         this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, 0L, masterHeartbeatInterval, TimeUnit.SECONDS);
-        logger.info("master node : {} registry to ZK successfully with heartBeatInterval : {}s",
-                    masterAddress,
-                    masterHeartbeatInterval);
+        logger.info("Master node : {} registered to registry center successfully with heartBeatInterval : {}s", masterAddress, masterHeartbeatInterval);
 
     }
 
@@ -223,12 +220,12 @@ public class MasterRegistryClient {
             String address = getLocalAddress();
             String localNodePath = getCurrentNodePath();
             registryClient.remove(localNodePath);
-            logger.info("master node : {} unRegistry to register center.", address);
+            logger.info("Master node : {} unRegistry to register center.", address);
             heartBeatExecutor.shutdown();
-            logger.info("heartbeat executor shutdown");
+            logger.info("MasterServer heartbeat executor shutdown");
             registryClient.close();
         } catch (Exception e) {
-            logger.error("remove registry path exception ", e);
+            logger.error("MasterServer remove registry path exception ", e);
         }
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
index 97c67d8493..3ce7d4e240 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/EventExecuteService.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.runner;
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
 
 import java.util.concurrent.TimeUnit;
@@ -49,25 +50,36 @@ public class EventExecuteService extends BaseDaemonThread {
 
     @Override
     public synchronized void start() {
+        logger.info("Master Event execute service starting");
         super.start();
+        logger.info("Master Event execute service started");
     }
 
     @Override
     public void run() {
-        logger.info("Event service started");
         while (Stopper.isRunning()) {
             try {
                 eventHandler();
                 TimeUnit.MILLISECONDS.sleep(Constants.SLEEP_TIME_MILLIS_SHORT);
+            } catch (InterruptedException interruptedException) {
+                logger.warn("Master event service interrupted, will exit this loop", interruptedException);
+                Thread.currentThread().interrupt();
+                break;
             } catch (Exception e) {
-                logger.error("Event service thread error", e);
+                logger.error("Master event execute service error", e);
             }
         }
     }
 
     private void eventHandler() {
         for (WorkflowExecuteRunnable workflowExecuteThread : this.processInstanceExecCacheManager.getAll()) {
-            workflowExecuteThreadPool.executeEvent(workflowExecuteThread);
+            try {
+                LoggerUtils.setWorkflowInstanceIdMDC(workflowExecuteThread.getProcessInstance().getId());
+                workflowExecuteThreadPool.executeEvent(workflowExecuteThread);
+
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
+            }
         }
     }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
index 1c6ea144e4..d6f3937f4b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/FailoverExecuteThread.java
@@ -49,7 +49,9 @@ public class FailoverExecuteThread extends BaseDaemonThread {
 
     @Override
     public synchronized void start() {
+        logger.info("Master failover thread staring");
         super.start();
+        logger.info("Master failover thread stared");
     }
 
     @Override
@@ -57,14 +59,13 @@ public class FailoverExecuteThread extends BaseDaemonThread {
         // when startup, wait 10s for ready
         ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * 10);
 
-        logger.info("failover execute thread started");
         while (Stopper.isRunning()) {
             try {
                 // todo: DO we need to schedule a task to do this kind of check
                 // This kind of check may only need to be executed when a master server start
                 failoverService.checkMasterFailover();
             } catch (Exception e) {
-                logger.error("failover execute error", e);
+                logger.error("Master failover thread execute error", e);
             } finally {
                 ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * masterConfig.getFailoverInterval() * 60);
             }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
index 94991a50c3..023d871355 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java
@@ -22,6 +22,7 @@ import org.apache.dolphinscheduler.common.enums.SlotCheckState;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.dao.entity.Command;
@@ -111,20 +112,23 @@ public class MasterSchedulerService extends BaseDaemonThread {
      * constructor of MasterSchedulerService
      */
     public void init() {
-        this.masterPrepareExecService = (ThreadPoolExecutor) ThreadUtils.newDaemonFixedThreadExecutor("Master-Pre-Exec-Thread", masterConfig.getPreExecThreads());
+        this.masterPrepareExecService = (ThreadPoolExecutor) ThreadUtils.newDaemonFixedThreadExecutor("MasterPreExecThread", masterConfig.getPreExecThreads());
         NettyClientConfig clientConfig = new NettyClientConfig();
         this.nettyRemotingClient = new NettyRemotingClient(clientConfig);
     }
 
     @Override
     public synchronized void start() {
+        logger.info("Master schedule service starting..");
         this.stateWheelExecuteThread.start();
         super.start();
+        logger.info("Master schedule service started...");
     }
 
     public void close() {
+        logger.info("Master schedule service stopping...");
         nettyRemotingClient.close();
-        logger.info("master schedule service stopped...");
+        logger.info("Master schedule service stopped...");
     }
 
     /**
@@ -132,7 +136,6 @@ public class MasterSchedulerService extends BaseDaemonThread {
      */
     @Override
     public void run() {
-        logger.info("master scheduler started");
         while (Stopper.isRunning()) {
             try {
                 boolean runCheckFlag = OSUtils.checkResource(masterConfig.getMaxCpuLoadAvg(), masterConfig.getReservedMemory());
@@ -142,8 +145,12 @@ public class MasterSchedulerService extends BaseDaemonThread {
                     continue;
                 }
                 scheduleProcess();
+            } catch (InterruptedException interruptedException) {
+                logger.warn("Master schedule service interrupted, close the loop", interruptedException);
+                Thread.currentThread().interrupt();
+                break;
             } catch (Exception e) {
-                logger.error("master scheduler thread error", e);
+                logger.error("Master schedule service loop command error", e);
             }
         }
     }
@@ -152,7 +159,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
      * 1. get command by slot
      * 2. donot handle command if slot is empty
      */
-    private void scheduleProcess() throws Exception {
+    private void scheduleProcess() throws InterruptedException {
         List<Command> commands = findCommands();
         if (CollectionUtils.isEmpty(commands)) {
             //indicate that no command ,sleep for 1s
@@ -167,8 +174,10 @@ public class MasterSchedulerService extends BaseDaemonThread {
         MasterServerMetrics.incMasterConsumeCommand(commands.size());
 
         for (ProcessInstance processInstance : processInstances) {
-
-            WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
+            try {
+                LoggerUtils.setWorkflowInstanceIdMDC(processInstance.getId());
+                logger.info("Master schedule service starting workflow instance");
+                WorkflowExecuteRunnable workflowExecuteRunnable = new WorkflowExecuteRunnable(
                     processInstance
                     , processService
                     , nettyExecutorManager
@@ -176,15 +185,21 @@ public class MasterSchedulerService extends BaseDaemonThread {
                     , masterConfig
                     , stateWheelExecuteThread);
 
-            this.processInstanceExecCacheManager.cache(processInstance.getId(), workflowExecuteRunnable);
-            if (processInstance.getTimeout() > 0) {
-                stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
+                this.processInstanceExecCacheManager.cache(processInstance.getId(), workflowExecuteRunnable);
+                if (processInstance.getTimeout() > 0) {
+                    stateWheelExecuteThread.addProcess4TimeoutCheck(processInstance);
+                }
+                workflowExecuteThreadPool.startWorkflow(workflowExecuteRunnable);
+                logger.info("Master schedule service started workflow instance");
+
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
             }
-            workflowExecuteThreadPool.startWorkflow(workflowExecuteRunnable);
         }
     }
 
-    private List<ProcessInstance> command2ProcessInstance(List<Command> commands) {
+    private List<ProcessInstance> command2ProcessInstance(List<Command> commands) throws InterruptedException {
+        logger.info("Master schedule service transforming command to ProcessInstance, commandSize: {}", commands.size());
         List<ProcessInstance> processInstances = Collections.synchronizedList(new ArrayList<>(commands.size()));
         CountDownLatch latch = new CountDownLatch(commands.size());
         for (final Command command : commands) {
@@ -193,7 +208,7 @@ public class MasterSchedulerService extends BaseDaemonThread {
                     // slot check again
                     SlotCheckState slotCheckState = slotCheck(command);
                     if (slotCheckState.equals(SlotCheckState.CHANGE) || slotCheckState.equals(SlotCheckState.INJECT)) {
-                        logger.info("handle command {} skip, slot check state: {}", command.getId(), slotCheckState);
+                        logger.info("Master handle command {} skip, slot check state: {}", command.getId(), slotCheckState);
                         return;
                     }
                     ProcessInstance processInstance = processService.handleCommand(logger,
@@ -201,10 +216,10 @@ public class MasterSchedulerService extends BaseDaemonThread {
                             command);
                     if (processInstance != null) {
                         processInstances.add(processInstance);
-                        logger.info("handle command {} end, create process instance {}", command.getId(), processInstance.getId());
+                        logger.info("Master handle command {} end, create process instance {}", command.getId(), processInstance.getId());
                     }
                 } catch (Exception e) {
-                    logger.error("handle command {} error ", command.getId(), e);
+                    logger.error("Master handle command {} error ", command.getId(), e);
                     processService.moveToErrorCommand(command, e.toString());
                 } finally {
                     latch.countDown();
@@ -212,13 +227,10 @@ public class MasterSchedulerService extends BaseDaemonThread {
             });
         }
 
-        try {
-            // make sure to finish handling command each time before next scan
-            latch.await();
-        } catch (InterruptedException e) {
-            logger.error("countDownLatch await error ", e);
-        }
-
+        // make sure to finish handling command each time before next scan
+        latch.await();
+        logger.info("Master schedule service transformed command to ProcessInstance, commandSize: {}, processInstanceSize: {}",
+            commands.size(), processInstances.size());
         return processInstances;
     }
 
@@ -231,6 +243,10 @@ public class MasterSchedulerService extends BaseDaemonThread {
             int masterCount = ServerNodeManager.getMasterSize();
             if (masterCount > 0) {
                 result = processService.findCommandPageBySlot(pageSize, pageNumber, masterCount, thisMasterSlot);
+                if (CollectionUtils.isNotEmpty(result)) {
+                    logger.info("Master schedule service loop command success, command size: {}, current slot: {}, total slot size: {}",
+                        result.size(), thisMasterSlot, masterCount);
+                }
             }
         }
         return result;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
index 12d404406c..53acb4ce83 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StateWheelExecuteThread.java
@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
 import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@@ -43,6 +44,8 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import lombok.NonNull;
+
 /**
  * Check thread
  * 1. timeout task check
@@ -110,10 +113,16 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
 
     public void addProcess4TimeoutCheck(ProcessInstance processInstance) {
         processInstanceTimeoutCheckList.add(processInstance.getId());
+        logger.info("Success add workflow instance into timeout check list");
     }
 
     public void removeProcess4TimeoutCheck(ProcessInstance processInstance) {
-        processInstanceTimeoutCheckList.remove(processInstance.getId());
+        boolean removeFlag = processInstanceTimeoutCheckList.remove(processInstance.getId());
+        if (removeFlag) {
+            logger.info("Success remove workflow instance from timeout check list");
+        } else {
+            logger.warn("Failed to remove workflow instance from timeout check list");
+        }
     }
 
     private void checkProcess4Timeout() {
@@ -121,106 +130,95 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
             return;
         }
         for (Integer processInstanceId : processInstanceTimeoutCheckList) {
-            if (processInstanceId == null) {
-                continue;
-            }
             WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
             if (workflowExecuteThread == null) {
-                logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}", processInstanceId);
+                logger.warn("Check workflow timeout failed, can not find workflowExecuteThread from cache manager, will remove this workflowInstance from check list");
                 processInstanceTimeoutCheckList.remove(processInstanceId);
                 continue;
             }
             ProcessInstance processInstance = workflowExecuteThread.getProcessInstance();
             if (processInstance == null) {
+                logger.warn("Check workflow timeout failed, the workflowInstance is null");
                 continue;
             }
             long timeRemain = DateUtils.getRemainTime(processInstance.getStartTime(), (long) processInstance.getTimeout() * Constants.SEC_2_MINUTES_TIME_UNIT);
             if (timeRemain < 0) {
+                logger.info("Workflow instance timeout, adding timeout event");
                 addProcessTimeoutEvent(processInstance);
                 processInstanceTimeoutCheckList.remove(processInstance.getId());
+                logger.info("Workflow instance timeout, added timeout event");
             }
         }
     }
 
-    public void addTask4TimeoutCheck(ProcessInstance processInstance, TaskInstance taskInstance) {
+    public void addTask4TimeoutCheck(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
         TaskInstanceKey taskInstanceKey = TaskInstanceKey.getTaskInstanceKey(processInstance, taskInstance);
-        if (taskInstanceKey == null) {
-            logger.error("taskInstanceKey is null");
-            return;
-        }
+        logger.info("Adding task instance into timeout check list");
         if (taskInstanceTimeoutCheckList.contains(taskInstanceKey)) {
+            logger.warn("Task instance is already in timeout check list");
             return;
         }
         TaskDefinition taskDefinition = taskInstance.getTaskDefine();
         if (taskDefinition == null) {
-            logger.error("taskDefinition is null, taskId:{}", taskInstance.getId());
+            logger.error("Failed to add task instance into timeout check list, taskDefinition is null");
             return;
         }
         if (TimeoutFlag.OPEN == taskDefinition.getTimeoutFlag()) {
             taskInstanceTimeoutCheckList.add(taskInstanceKey);
+            logger.info("Timeout flag is open, added task instance into timeout check list");
         }
         if (taskInstance.isDependTask() || taskInstance.isSubProcess()) {
             taskInstanceTimeoutCheckList.add(taskInstanceKey);
+            logger.info("task instance is dependTask orSubProcess, added task instance into timeout check list");
         }
     }
 
-    public void removeTask4TimeoutCheck(ProcessInstance processInstance, TaskInstance taskInstance) {
+    public void removeTask4TimeoutCheck(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
         TaskInstanceKey taskInstanceKey = TaskInstanceKey.getTaskInstanceKey(processInstance, taskInstance);
-        if (taskInstanceKey == null) {
-            logger.error("taskInstanceKey is null");
-            return;
-        }
         taskInstanceTimeoutCheckList.remove(taskInstanceKey);
+        logger.info("remove task instance from timeout check list");
     }
 
-    public void addTask4RetryCheck(ProcessInstance processInstance, TaskInstance taskInstance) {
+    public void addTask4RetryCheck(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
+        logger.info("Adding task instance into retry check list");
         TaskInstanceKey taskInstanceKey = TaskInstanceKey.getTaskInstanceKey(processInstance, taskInstance);
-        if (taskInstanceKey == null) {
-            logger.error("taskInstanceKey is null");
-            return;
-        }
         if (taskInstanceRetryCheckList.contains(taskInstanceKey)) {
+            logger.warn("Task instance is already in retry check list");
             return;
         }
         TaskDefinition taskDefinition = taskInstance.getTaskDefine();
         if (taskDefinition == null) {
-            logger.error("taskDefinition is null, taskId:{}", taskInstance.getId());
+            logger.error("Add task instance into retry check list error, taskDefinition is null");
             return;
         }
-        logger.debug("addTask4RetryCheck, taskCode:{}, processInstanceId:{}", taskInstance.getTaskCode(), taskInstance.getProcessInstanceId());
         taskInstanceRetryCheckList.add(taskInstanceKey);
+        logger.info("[WorkflowInstance-{}][TaskInstance-{}] Added task instance into retry check list",
+            processInstance.getId(), taskInstance.getId());
     }
 
-    public void removeTask4RetryCheck(ProcessInstance processInstance, TaskInstance taskInstance) {
+    public void removeTask4RetryCheck(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
         TaskInstanceKey taskInstanceKey = TaskInstanceKey.getTaskInstanceKey(processInstance, taskInstance);
-        if (taskInstanceKey == null) {
-            logger.error("taskInstanceKey is null");
-            return;
-        }
         taskInstanceRetryCheckList.remove(taskInstanceKey);
+        logger.info("remove task instance from retry check list");
     }
 
-    public void addTask4StateCheck(ProcessInstance processInstance, TaskInstance taskInstance) {
+    public void addTask4StateCheck(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
+        logger.info("Adding task instance into state check list");
         TaskInstanceKey taskInstanceKey = TaskInstanceKey.getTaskInstanceKey(processInstance, taskInstance);
-        if (taskInstanceKey == null) {
-            logger.error("taskInstanceKey is null");
-            return;
-        }
         if (taskInstanceStateCheckList.contains(taskInstanceKey)) {
+            logger.warn("Task instance is already in state check list");
             return;
         }
         if (taskInstance.isDependTask() || taskInstance.isSubProcess()) {
             taskInstanceStateCheckList.add(taskInstanceKey);
+            logger.info("Added task instance into state check list");
         }
     }
 
-    public void removeTask4StateCheck(ProcessInstance processInstance, TaskInstance taskInstance) {
+    public void removeTask4StateCheck(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
         TaskInstanceKey taskInstanceKey = TaskInstanceKey.getTaskInstanceKey(processInstance, taskInstance);
-        if (taskInstanceKey == null) {
-            logger.error("taskInstanceKey is null");
-            return;
-        }
         taskInstanceStateCheckList.remove(taskInstanceKey);
+        logger.info("Removed task instance from state check list");
     }
 
     private void checkTask4Timeout() {
@@ -228,30 +226,35 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
             return;
         }
         for (TaskInstanceKey taskInstanceKey : taskInstanceTimeoutCheckList) {
-            int processInstanceId = taskInstanceKey.getProcessInstanceId();
-            long taskCode = taskInstanceKey.getTaskCode();
+            try {
+                int processInstanceId = taskInstanceKey.getProcessInstanceId();
+                LoggerUtils.setWorkflowInstanceIdMDC(processInstanceId);
+                long taskCode = taskInstanceKey.getTaskCode();
 
-            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
-            if (workflowExecuteThread == null) {
-                logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
-                        processInstanceId, taskCode);
-                taskInstanceTimeoutCheckList.remove(taskInstanceKey);
-                continue;
-            }
-            Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
-            if (!taskInstanceOptional.isPresent()) {
-                logger.warn("can not find taskInstance from workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
-                        processInstanceId, taskCode);
-                taskInstanceTimeoutCheckList.remove(taskInstanceKey);
-                continue;
-            }
-            TaskInstance taskInstance = taskInstanceOptional.get();
-            if (TimeoutFlag.OPEN == taskInstance.getTaskDefine().getTimeoutFlag()) {
-                long timeRemain = DateUtils.getRemainTime(taskInstance.getStartTime(), (long) taskInstance.getTaskDefine().getTimeout() * Constants.SEC_2_MINUTES_TIME_UNIT);
-                if (timeRemain < 0) {
-                    addTaskTimeoutEvent(taskInstance);
+                WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+                if (workflowExecuteThread == null) {
+                    logger.warn("Check task instance timeout failed, can not find workflowExecuteThread from cache manager, will remove this check task");
+                    taskInstanceTimeoutCheckList.remove(taskInstanceKey);
+                    continue;
+                }
+                Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
+                if (!taskInstanceOptional.isPresent()) {
+                    logger.warn("Check task instance timeout failed, can not get taskInstance from workflowExecuteThread, taskCode: {}"
+                        + "will remove this check task", taskCode);
                     taskInstanceTimeoutCheckList.remove(taskInstanceKey);
+                    continue;
+                }
+                TaskInstance taskInstance = taskInstanceOptional.get();
+                if (TimeoutFlag.OPEN == taskInstance.getTaskDefine().getTimeoutFlag()) {
+                    long timeRemain = DateUtils.getRemainTime(taskInstance.getStartTime(), (long) taskInstance.getTaskDefine().getTimeout() * Constants.SEC_2_MINUTES_TIME_UNIT);
+                    if (timeRemain < 0) {
+                        logger.info("Task instance is timeout, adding task timeout event and remove the check");
+                        addTaskTimeoutEvent(taskInstance);
+                        taskInstanceTimeoutCheckList.remove(taskInstanceKey);
+                    }
                 }
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
             }
         }
     }
@@ -264,41 +267,46 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
         for (TaskInstanceKey taskInstanceKey : taskInstanceRetryCheckList) {
             int processInstanceId = taskInstanceKey.getProcessInstanceId();
             long taskCode = taskInstanceKey.getTaskCode();
+            try {
+                LoggerUtils.setWorkflowInstanceIdMDC(processInstanceId);
 
-            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+                WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
 
-            if (workflowExecuteThread == null) {
-                logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
-                        processInstanceId, taskCode);
-                taskInstanceRetryCheckList.remove(taskInstanceKey);
-                continue;
-            }
+                if (workflowExecuteThread == null) {
+                    logger.warn("Task instance retry check failed, can not find workflowExecuteThread from cache manager, "
+                        + "will remove this check task");
+                    taskInstanceRetryCheckList.remove(taskInstanceKey);
+                    continue;
+                }
 
-            Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getRetryTaskInstanceByTaskCode(taskCode);
-            ProcessInstance processInstance = workflowExecuteThread.getProcessInstance();
+                Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getRetryTaskInstanceByTaskCode(taskCode);
+                ProcessInstance processInstance = workflowExecuteThread.getProcessInstance();
 
-            if (processInstance.getState() == ExecutionStatus.READY_STOP) {
-                addProcessStopEvent(processInstance);
-                taskInstanceRetryCheckList.remove(taskInstanceKey);
-                break;
-            }
+                if (processInstance.getState() == ExecutionStatus.READY_STOP) {
+                    logger.warn("The process instance is ready to stop, will send process stop event and remove the check task");
+                    addProcessStopEvent(processInstance);
+                    taskInstanceRetryCheckList.remove(taskInstanceKey);
+                    break;
+                }
 
-            if (!taskInstanceOptional.isPresent()) {
-                logger.warn("can not find taskInstance from workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
-                        processInstanceId, taskCode);
-                taskInstanceRetryCheckList.remove(taskInstanceKey);
-                continue;
-            }
+                if (!taskInstanceOptional.isPresent()) {
+                    logger.warn("Task instance retry check failed, can not find taskInstance from workflowExecuteThread, will remove this check");
+                    taskInstanceRetryCheckList.remove(taskInstanceKey);
+                    continue;
+                }
 
-            TaskInstance taskInstance = taskInstanceOptional.get();
-            if (taskInstance.retryTaskIntervalOverTime()) {
-                // reset taskInstance endTime and state
-                // todo relative funtion: TaskInstance.retryTaskIntervalOverTime, WorkflowExecuteThread.cloneRetryTaskInstance
-                taskInstance.setEndTime(null);
-                taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
+                TaskInstance taskInstance = taskInstanceOptional.get();
+                if (taskInstance.retryTaskIntervalOverTime()) {
+                    // reset taskInstance endTime and state
+                    // todo relative funtion: TaskInstance.retryTaskIntervalOverTime, WorkflowExecuteThread.cloneRetryTaskInstance
+                    taskInstance.setEndTime(null);
+                    taskInstance.setState(ExecutionStatus.SUBMITTED_SUCCESS);
 
-                addTaskRetryEvent(taskInstance);
-                taskInstanceRetryCheckList.remove(taskInstanceKey);
+                    addTaskRetryEvent(taskInstance);
+                    taskInstanceRetryCheckList.remove(taskInstanceKey);
+                }
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
             }
         }
     }
@@ -311,25 +319,29 @@ public class StateWheelExecuteThread extends BaseDaemonThread {
             int processInstanceId = taskInstanceKey.getProcessInstanceId();
             long taskCode = taskInstanceKey.getTaskCode();
 
-            WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
-            if (workflowExecuteThread == null) {
-                logger.warn("can not find workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
-                        processInstanceId, taskCode);
-                taskInstanceStateCheckList.remove(taskInstanceKey);
-                continue;
-            }
-            Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
-            if (!taskInstanceOptional.isPresent()) {
-                logger.warn("can not find taskInstance from workflowExecuteThread, this check event will remove, processInstanceId:{}, taskCode:{}",
-                        processInstanceId, taskCode);
-                taskInstanceStateCheckList.remove(taskInstanceKey);
-                continue;
-            }
-            TaskInstance taskInstance = taskInstanceOptional.get();
-            if (taskInstance.getState().typeIsFinished()) {
-                continue;
+            try {
+                LoggerUtils.setTaskInstanceIdMDC(processInstanceId);
+                WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(processInstanceId);
+                if (workflowExecuteThread == null) {
+                    logger.warn("Task instance state check failed, can not find workflowExecuteThread from cache manager, will remove this check task");
+                    taskInstanceStateCheckList.remove(taskInstanceKey);
+                    continue;
+                }
+                Optional<TaskInstance> taskInstanceOptional = workflowExecuteThread.getActiveTaskInstanceByTaskCode(taskCode);
+                if (!taskInstanceOptional.isPresent()) {
+                    logger.warn(
+                        "Task instance state check failed, can not find taskInstance from workflowExecuteThread, will remove this check event");
+                    taskInstanceStateCheckList.remove(taskInstanceKey);
+                    continue;
+                }
+                TaskInstance taskInstance = taskInstanceOptional.get();
+                if (taskInstance.getState().typeIsFinished()) {
+                    continue;
+                }
+                addTaskStateChangeEvent(taskInstance);
+            } finally {
+                LoggerUtils.removeWorkflowInstanceIdMDC();
             }
-            addTaskStateChangeEvent(taskInstance);
         }
     }
 
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 07e0f14c11..9c5f4062f7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -43,6 +43,7 @@ import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
 import org.apache.dolphinscheduler.common.process.ProcessDag;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.common.utils.ParameterUtils;
 import org.apache.dolphinscheduler.dao.entity.Command;
@@ -271,12 +272,16 @@ public class WorkflowExecuteRunnable implements Runnable {
         while (!this.stateEvents.isEmpty()) {
             try {
                 StateEvent stateEvent = this.stateEvents.peek();
+                LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
                 if (stateEventHandler(stateEvent)) {
                     this.stateEvents.remove(stateEvent);
                 }
             } catch (Exception e) {
                 logger.error("state handle error:", e);
+            } finally {
+                LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
             }
+
         }
     }
 
@@ -775,7 +780,16 @@ public class WorkflowExecuteRunnable implements Runnable {
         if (cmdParam.containsKey(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING)) {
             cmdParam.remove(Constants.CMD_PARAM_RECOVERY_START_NODE_STRING);
         }
-        cmdParam.replace(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.format(scheduleDate, "yyyy-MM-dd HH:mm:ss", null));
+
+        if (cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)) {
+            cmdParam.replace(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST,
+                cmdParam.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST)
+                    .substring(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST).indexOf(COMMA) + 1));
+        }
+
+        if (cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE)) {
+            cmdParam.replace(CMDPARAM_COMPLEMENT_DATA_START_DATE, DateUtils.format(scheduleDate, YYYY_MM_DD_HH_MM_SS, null));
+        }
         command.setCommandParam(JSONUtils.toJsonString(cmdParam));
         command.setTaskDependType(processInstance.getTaskDependType());
         command.setFailureStrategy(processInstance.getFailureStrategy());
@@ -962,8 +976,12 @@ public class WorkflowExecuteRunnable implements Runnable {
                 // reset global params while there are start parameters
                 setGlobalParamIfCommanded(processDefinition, cmdParam);
 
-                Date start = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
-                Date end = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
+                Date start = null;
+                Date end = null;
+                if(cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_START_DATE) && cmdParam.containsKey(CMDPARAM_COMPLEMENT_DATA_END_DATE)){
+                    start = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_START_DATE));
+                    end = DateUtils.stringToDate(cmdParam.get(CMDPARAM_COMPLEMENT_DATA_END_DATE));
+                }
                 List<Schedule> schedules = processService.queryReleaseSchedulerListByProcessDefinitionCode(processInstance.getProcessDefinitionCode());
                 if (complementListDate.isEmpty() && needComplementProcess()) {
                     complementListDate = CronUtils.getSelfFireDateList(start, end, schedules);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
index 92ff7c04f1..031f8986ea 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThreadPool.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.server.master.runner;
 import org.apache.dolphinscheduler.common.enums.Flag;
 import org.apache.dolphinscheduler.common.enums.StateEvent;
 import org.apache.dolphinscheduler.common.enums.StateEventType;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
 import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@@ -48,6 +49,8 @@ import org.springframework.util.concurrent.ListenableFutureCallback;
 
 import com.google.common.base.Strings;
 
+import lombok.NonNull;
+
 /**
  * Used to execute {@link WorkflowExecuteRunnable}, when
  */
@@ -79,7 +82,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     @PostConstruct
     private void init() {
         this.setDaemon(true);
-        this.setThreadNamePrefix("Workflow-Execute-Thread-");
+        this.setThreadNamePrefix("WorkflowExecuteThread-");
         this.setMaxPoolSize(masterConfig.getExecThreads());
         this.setCorePoolSize(masterConfig.getExecThreads());
     }
@@ -90,10 +93,11 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     public void submitStateEvent(StateEvent stateEvent) {
         WorkflowExecuteRunnable workflowExecuteThread = processInstanceExecCacheManager.getByProcessInstanceId(stateEvent.getProcessInstanceId());
         if (workflowExecuteThread == null) {
-            logger.warn("workflowExecuteThread is null, stateEvent:{}", stateEvent);
+            logger.warn("Submit state event error, cannot from workflowExecuteThread from cache manager, stateEvent:{}", stateEvent);
             return;
         }
         workflowExecuteThread.addStateEvent(stateEvent);
+        logger.info("Submit state event success, stateEvent: {}", stateEvent);
     }
 
     /**
@@ -112,7 +116,7 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
             return;
         }
         if (multiThreadFilterMap.containsKey(workflowExecuteThread.getKey())) {
-            logger.warn("The workflow:{} has been executed by another thread", workflowExecuteThread.getKey());
+            logger.warn("The workflow has been executed by another thread");
             return;
         }
         multiThreadFilterMap.put(workflowExecuteThread.getKey(), workflowExecuteThread);
@@ -121,24 +125,31 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
         future.addCallback(new ListenableFutureCallback() {
             @Override
             public void onFailure(Throwable ex) {
-                logger.error("handle events {} failed", processInstanceId, ex);
-                multiThreadFilterMap.remove(workflowExecuteThread.getKey());
+                LoggerUtils.setWorkflowInstanceIdMDC(processInstanceId);
+                try {
+                    logger.error("Workflow instance events handle failed", ex);
+                    multiThreadFilterMap.remove(workflowExecuteThread.getKey());
+                } finally {
+                    LoggerUtils.removeWorkflowInstanceIdMDC();
+                }
             }
 
             @Override
             public void onSuccess(Object result) {
                 try {
+                    LoggerUtils.setWorkflowInstanceIdMDC(workflowExecuteThread.getProcessInstance().getId());
                     if (workflowExecuteThread.workFlowFinish()) {
                         stateWheelExecuteThread.removeProcess4TimeoutCheck(workflowExecuteThread.getProcessInstance());
                         processInstanceExecCacheManager.removeByProcessInstanceId(processInstanceId);
                         notifyProcessChanged(workflowExecuteThread.getProcessInstance());
-                        logger.info("process instance {} finished.", processInstanceId);
+                        logger.info("Workflow instance is finished.");
                     }
                 } catch (Exception e) {
-                    logger.error("handle events {} success, but notify changed error", processInstanceId, e);
+                    logger.error("Workflow instance is finished, but notify changed error", e);
                 } finally {
                     // make sure the process has been removed from multiThreadFilterMap
                     multiThreadFilterMap.remove(workflowExecuteThread.getKey());
+                    LoggerUtils.removeWorkflowInstanceIdMDC();
                 }
             }
         });
@@ -167,9 +178,9 @@ public class WorkflowExecuteThreadPool extends ThreadPoolTaskExecutor {
     /**
      * notify myself
      */
-    private void notifyMyself(ProcessInstance processInstance, TaskInstance taskInstance) {
-        logger.info("notify process {} task {} state change", processInstance.getId(), taskInstance.getId());
+    private void notifyMyself(@NonNull ProcessInstance processInstance, @NonNull TaskInstance taskInstance) {
         if (!processInstanceExecCacheManager.contains(processInstance.getId())) {
+            logger.warn("The execute cache manager doesn't contains this workflow instance");
             return;
         }
         StateEvent stateEvent = new StateEvent();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
index 41c2bd56d3..4c732a0325 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/TaskProcessorFactory.java
@@ -30,9 +30,12 @@ import java.util.concurrent.ConcurrentHashMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import lombok.experimental.UtilityClass;
+
 /**
  * the factory to create task processor
  */
+@UtilityClass
 public final class TaskProcessorFactory {
 
     private static final Logger logger = LoggerFactory.getLogger(TaskProcessorFactory.class);
@@ -46,7 +49,7 @@ public final class TaskProcessorFactory {
             try {
                 PROCESS_MAP.put(iTaskProcessor.getType(), (Constructor<ITaskProcessor>) iTaskProcessor.getClass().getConstructor());
             } catch (NoSuchMethodException e) {
-                throw new IllegalArgumentException("The task processor should has a no args constructor");
+                throw new IllegalArgumentException("The task processor should has a no args constructor", e);
             }
         }
     }
@@ -57,7 +60,6 @@ public final class TaskProcessorFactory {
         }
         Constructor<ITaskProcessor> iTaskProcessorConstructor = PROCESS_MAP.get(type);
         if (iTaskProcessorConstructor == null) {
-            logger.warn("ITaskProcessor could not found for taskType: {}", type);
             iTaskProcessorConstructor = PROCESS_MAP.get(DEFAULT_PROCESSOR);
         }
 
@@ -74,7 +76,4 @@ public final class TaskProcessorFactory {
         return PROCESS_MAP.containsKey(type);
     }
 
-    private TaskProcessorFactory() {
-        throw new UnsupportedOperationException("TaskProcessorFactory cannot be instantiated");
-    }
 }
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
index 3cc92fb905..94f33ccd83 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/service/FailoverService.java
@@ -17,8 +17,7 @@
 
 package org.apache.dolphinscheduler.server.master.service;
 
-import io.micrometer.core.annotation.Counted;
-import io.micrometer.core.annotation.Timed;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.enums.NodeType;
@@ -55,6 +54,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
+import io.micrometer.core.annotation.Counted;
+import io.micrometer.core.annotation.Timed;
+
 /**
  * failover service
  */
@@ -66,12 +68,14 @@ public class FailoverService {
     private final ProcessService processService;
     private final WorkflowExecuteThreadPool workflowExecuteThreadPool;
 
-    public FailoverService(RegistryClient registryClient, MasterConfig masterConfig, ProcessService processService,
+    public FailoverService(RegistryClient registryClient,
+                           MasterConfig masterConfig,
+                           ProcessService processService,
                            WorkflowExecuteThreadPool workflowExecuteThreadPool) {
-        this.registryClient = registryClient;
-        this.masterConfig = masterConfig;
-        this.processService = processService;
-        this.workflowExecuteThreadPool = workflowExecuteThreadPool;
+        this.registryClient = checkNotNull(registryClient);
+        this.masterConfig = checkNotNull(masterConfig);
+        this.processService = checkNotNull(processService);
+        this.workflowExecuteThreadPool = checkNotNull(workflowExecuteThreadPool);
     }
 
     /**
@@ -84,7 +88,7 @@ public class FailoverService {
         if (CollectionUtils.isEmpty(hosts)) {
             return;
         }
-        LOGGER.info("{} begin to failover hosts:{}", getLocalAddress(), hosts);
+        LOGGER.info("Master failover service {} begin to failover hosts:{}", getLocalAddress(), hosts);
 
         for (String host : hosts) {
             failoverMasterWithLock(host);
@@ -274,7 +278,7 @@ public class FailoverService {
         while (iterator.hasNext()) {
             String host = iterator.next();
             if (registryClient.checkNodeExists(host, NodeType.MASTER)) {
-                if (!host.equals(getLocalAddress())) {
+                if (!getLocalAddress().equals(host)) {
                     iterator.remove();
                 }
             }
@@ -294,7 +298,7 @@ public class FailoverService {
         boolean taskNeedFailover = true;
 
         if (taskInstance == null) {
-            LOGGER.error("failover task instance error, taskInstance is null");
+            LOGGER.error("Master failover task instance error, taskInstance is null");
             return false;
         }
 
diff --git a/dolphinscheduler-master/src/main/resources/logback-spring.xml b/dolphinscheduler-master/src/main/resources/logback-spring.xml
index 2e9cb45ada..f1a2c1d51b 100644
--- a/dolphinscheduler-master/src/main/resources/logback-spring.xml
+++ b/dolphinscheduler-master/src/main/resources/logback-spring.xml
@@ -21,7 +21,7 @@
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>
-                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n
             </pattern>
             <charset>UTF-8</charset>
         </encoder>
@@ -57,7 +57,7 @@
         </rollingPolicy>
         <encoder>
             <pattern>
-                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n
             </pattern>
             <charset>UTF-8</charset>
         </encoder>
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java
index f5d7f935b3..ce6ad57035 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java
@@ -25,6 +25,8 @@ import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
 import org.apache.dolphinscheduler.remote.utils.Host;
 import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
 
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -36,7 +38,7 @@ public class AlertClientService implements AutoCloseable {
 
     private final NettyRemotingClient client;
 
-    private volatile boolean isRunning;
+    private final AtomicBoolean isRunning;
 
     private String host;
 
@@ -53,16 +55,14 @@ public class AlertClientService implements AutoCloseable {
     public AlertClientService() {
         this.clientConfig = new NettyClientConfig();
         this.client = new NettyRemotingClient(clientConfig);
-        this.isRunning = true;
+        this.isRunning = new AtomicBoolean(true);
     }
 
     /**
      * alert client
      */
     public AlertClientService(String host, int port) {
-        this.clientConfig = new NettyClientConfig();
-        this.client = new NettyRemotingClient(clientConfig);
-        this.isRunning = true;
+        this();
         this.host = host;
         this.port = port;
     }
@@ -72,9 +72,14 @@ public class AlertClientService implements AutoCloseable {
      */
     @Override
     public void close() {
+        if (isRunning.compareAndSet(true, false)) {
+            logger.warn("Alert client is already closed");
+            return;
+        }
+
+        logger.info("Alter client closing");
         this.client.close();
-        this.isRunning = false;
-        logger.info("alter client closed");
+        logger.info("Alter client closed");
     }
 
     /**
@@ -116,6 +121,6 @@ public class AlertClientService implements AutoCloseable {
     }
 
     public boolean isRunning() {
-        return isRunning;
+        return isRunning.get();
     }
 }
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java
index 7dffe9c204..289e8ddbfb 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/task/TaskPluginManager.java
@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.dao.PluginDao;
 import org.apache.dolphinscheduler.dao.entity.PluginDefine;
 import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
 import org.apache.dolphinscheduler.plugin.task.api.TaskChannelFactory;
+import org.apache.dolphinscheduler.plugin.task.api.TaskPluginException;
 import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
 import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode;
 import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
@@ -93,7 +94,7 @@ public class TaskPluginManager {
             logger.info("Registering task plugin: {}", name);
 
             if (!names.add(name)) {
-                throw new IllegalStateException(format("Duplicate task plugins named '%s'", name));
+                throw new TaskPluginException(format("Duplicate task plugins named '%s'", name));
             }
 
             loadTaskChannel(factory);
@@ -106,7 +107,7 @@ public class TaskPluginManager {
             PluginDefine pluginDefine = new PluginDefine(name, PluginType.TASK.getDesc(), paramsJson);
             int count = pluginDao.addOrUpdatePluginDefine(pluginDefine);
             if (count <= 0) {
-                throw new RuntimeException("Failed to update task plugin: " + name);
+                throw new TaskPluginException("Failed to update task plugin: " + name);
             }
         });
     }
diff --git a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
index d6ef76fa9e..0cdeabde5e 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
+++ b/dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml
@@ -22,7 +22,7 @@
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>
-                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n
             </pattern>
             <charset>UTF-8</charset>
         </encoder>
@@ -63,7 +63,7 @@
                 <file>${log.base}/${taskAppId}.log</file>
                 <encoder>
                     <pattern>
-                        [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{96}:[%line] - %messsage%n
+                        [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} [%thread] %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %messsage%n
                     </pattern>
                     <charset>UTF-8</charset>
                 </encoder>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java
index d052075988..528dfa4058 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/ProcessUtils.java
@@ -25,6 +25,8 @@ import java.util.regex.Pattern;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import lombok.NonNull;
+
 public final class ProcessUtils {
     private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
 
@@ -46,13 +48,13 @@ public final class ProcessUtils {
     /**
      * kill tasks according to different task types.
      */
-    public static void kill(TaskExecutionContext request) {
+    public static boolean kill(@NonNull TaskExecutionContext request) {
         try {
+            logger.info("Begin kill task instance, processId: {}", request.getProcessId());
             int processId = request.getProcessId();
             if (processId == 0) {
-                logger.error("process kill failed, process id :{}, task id:{}",
-                        processId, request.getTaskInstanceId());
-                return;
+                logger.error("Task instance kill failed, processId is not exist");
+                return false;
             }
 
             String cmd = String.format("kill -9 %s", getPidsStr(processId));
@@ -60,8 +62,11 @@ public final class ProcessUtils {
             logger.info("process id:{}, cmd:{}", processId, cmd);
 
             OSUtils.exeCmd(cmd);
+            logger.info("Success kill task instance, processId: {}", request.getProcessId());
+            return true;
         } catch (Exception e) {
-            logger.error("kill task failed", e);
+            logger.error("Kill task instance error, processId: {}", request.getProcessId(), e);
+            return false;
         }
     }
 
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginException.java
similarity index 60%
copy from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java
copy to dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginException.java
index 67abde7e7a..246835260b 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/Stopper.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskPluginException.java
@@ -15,26 +15,11 @@
  * limitations under the License.
  */
 
-package org.apache.dolphinscheduler.common.thread;
+package org.apache.dolphinscheduler.plugin.task.api;
 
-import java.util.concurrent.atomic.AtomicBoolean;
+public class TaskPluginException extends RuntimeException {
 
-/**
- *  if the process closes, a signal is placed as true, and all threads get this flag to stop working
- */
-public class Stopper {
-
-    private static AtomicBoolean signal = new AtomicBoolean(false);
-
-    public static final boolean isStopped() {
-        return signal.get();
-    }
-
-    public static final boolean isRunning() {
-        return !signal.get();
-    }
-
-    public static final void stop() {
-        signal.set(true);
+    public TaskPluginException(String message) {
+        super(message);
     }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
index 8ac528dee4..93dc6309db 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java
@@ -21,18 +21,11 @@ import org.apache.dolphinscheduler.common.Constants;
 import org.apache.dolphinscheduler.common.IStoppable;
 import org.apache.dolphinscheduler.common.enums.NodeType;
 import org.apache.dolphinscheduler.common.thread.Stopper;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.plugin.task.api.ProcessUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
-import org.apache.dolphinscheduler.remote.NettyRemotingServer;
-import org.apache.dolphinscheduler.remote.command.CommandType;
-import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
-import org.apache.dolphinscheduler.server.log.LoggerRequestProcessor;
-import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
-import org.apache.dolphinscheduler.server.worker.processor.HostUpdateProcessor;
-import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor;
-import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteResponseAckProcessor;
-import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteRunningAckProcessor;
-import org.apache.dolphinscheduler.server.worker.processor.TaskKillProcessor;
+import org.apache.dolphinscheduler.server.worker.prc.WorkerRpcServer;
 import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient;
 import org.apache.dolphinscheduler.server.worker.runner.RetryReportTaskStatusThread;
 import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
@@ -65,17 +58,6 @@ public class WorkerServer implements IStoppable {
      */
     private static final Logger logger = LoggerFactory.getLogger(WorkerServer.class);
 
-    /**
-     * netty remote server
-     */
-    private NettyRemotingServer nettyRemotingServer;
-
-    /**
-     * worker config
-     */
-    @Autowired
-    private WorkerConfig workerConfig;
-
     /**
      * spring application context
      * only use it for initialization
@@ -105,22 +87,7 @@ public class WorkerServer implements IStoppable {
     private TaskPluginManager taskPluginManager;
 
     @Autowired
-    private TaskExecuteProcessor taskExecuteProcessor;
-
-    @Autowired
-    private TaskKillProcessor taskKillProcessor;
-
-    @Autowired
-    private TaskExecuteRunningAckProcessor taskExecuteRunningAckProcessor;
-
-    @Autowired
-    private TaskExecuteResponseAckProcessor taskExecuteResponseAckProcessor;
-
-    @Autowired
-    private HostUpdateProcessor hostUpdateProcessor;
-
-    @Autowired
-    private LoggerRequestProcessor loggerRequestProcessor;
+    private WorkerRpcServer workerRpcServer;
 
     /**
      * worker server startup, not use web service
@@ -132,48 +99,19 @@ public class WorkerServer implements IStoppable {
         SpringApplication.run(WorkerServer.class);
     }
 
-    /**
-     * worker server run
-     */
     @PostConstruct
     public void run() {
-        // init remoting server
-        NettyServerConfig serverConfig = new NettyServerConfig();
-        serverConfig.setListenPort(workerConfig.getListenPort());
-        this.nettyRemotingServer = new NettyRemotingServer(serverConfig);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_REQUEST, taskExecuteProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_REQUEST, taskKillProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RUNNING_ACK, taskExecuteRunningAckProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE_ACK, taskExecuteResponseAckProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.PROCESS_HOST_UPDATE_REQUEST, hostUpdateProcessor);
-
-        // logger server
-        this.nettyRemotingServer.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, loggerRequestProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.ROLL_VIEW_LOG_REQUEST, loggerRequestProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.VIEW_WHOLE_LOG_REQUEST, loggerRequestProcessor);
-        this.nettyRemotingServer.registerProcessor(CommandType.REMOVE_TAK_LOG_REQUEST, loggerRequestProcessor);
-
-        this.nettyRemotingServer.start();
-
-        // install task plugin
-        this.taskPluginManager.installPlugin();
+        this.workerRpcServer.start();
 
-        // worker registry
-        try {
-            this.workerRegistryClient.registry();
-            this.workerRegistryClient.setRegistryStoppable(this);
-            Set<String> workerZkPaths = this.workerRegistryClient.getWorkerZkPaths();
+        this.taskPluginManager.installPlugin();
 
-            this.workerRegistryClient.handleDeadServer(workerZkPaths, NodeType.WORKER, Constants.DELETE_OP);
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-            throw new RuntimeException(e);
-        }
+        this.workerRegistryClient.registry();
+        this.workerRegistryClient.setRegistryStoppable(this);
+        Set<String> workerZkPaths = this.workerRegistryClient.getWorkerZkPaths();
+        this.workerRegistryClient.handleDeadServer(workerZkPaths, NodeType.WORKER, Constants.DELETE_OP);
 
-        // task execute manager
         this.workerManagerThread.start();
 
-        // retry report task status
         this.retryReportTaskStatusThread.start();
 
         /*
@@ -181,7 +119,7 @@ public class WorkerServer implements IStoppable {
          */
         Runtime.getRuntime().addShutdownHook(new Thread(() -> {
             if (Stopper.isRunning()) {
-                close("shutdownHook");
+                close("WorkerServer shutdown hook");
             }
         }));
     }
@@ -189,24 +127,23 @@ public class WorkerServer implements IStoppable {
     public void close(String cause) {
         try {
             // execute only once
-            if (Stopper.isStopped()) {
+            // set stop signal is true
+            if (!Stopper.stop()) {
+                logger.warn("WorkerServer is already stopped, current cause: {}", cause);
                 return;
             }
 
-            logger.info("worker server is stopping ..., cause : {}", cause);
-
-            // set stop signal is true
-            Stopper.stop();
+            logger.info("Worker server is stopping, current cause : {}", cause);
 
             try {
                 // thread sleep 3 seconds for thread quitely stop
-                Thread.sleep(3000L);
+                Thread.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
             } catch (Exception e) {
-                logger.warn("thread sleep exception", e);
+                logger.warn("Worker server close wait error", e);
             }
 
             // close
-            this.nettyRemotingServer.close();
+            this.workerRpcServer.close();
             this.workerRegistryClient.unRegistry();
             this.alertClientService.close();
 
@@ -215,8 +152,9 @@ public class WorkerServer implements IStoppable {
 
             // close the application context
             this.springApplicationContext.close();
+            logger.info("Worker server stopped, current cause: {}", cause);
         } catch (Exception e) {
-            logger.error("worker server stop exception ", e);
+            logger.error("Worker server stop failed, current cause: {}", cause, e);
         }
     }
 
@@ -230,15 +168,22 @@ public class WorkerServer implements IStoppable {
      */
     public void killAllRunningTasks() {
         Collection<TaskExecutionContext> taskRequests = TaskExecutionContextCacheManager.getAllTaskRequestList();
-        logger.info("ready to kill all cache job, job size:{}", taskRequests.size());
-
         if (CollectionUtils.isEmpty(taskRequests)) {
             return;
         }
-
+        logger.info("Worker begin to kill all cache task, task size: {}", taskRequests.size());
+        int killNumber = 0;
         for (TaskExecutionContext taskRequest : taskRequests) {
             // kill task when it's not finished yet
-            org.apache.dolphinscheduler.plugin.task.api.ProcessUtils.kill(taskRequest);
+            try {
+                LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskRequest.getProcessInstanceId(), taskRequest.getTaskInstanceId());
+                if (ProcessUtils.kill(taskRequest)) {
+                    killNumber++;
+                }
+            } finally {
+                LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
+            }
         }
+        logger.info("Worker after kill all cache task, task size: {}, killed number: {}", taskRequests.size(), killNumber);
     }
 }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/prc/WorkerRpcServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/prc/WorkerRpcServer.java
new file mode 100644
index 0000000000..acb0ac6508
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/prc/WorkerRpcServer.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.prc;
+
+import org.apache.dolphinscheduler.remote.NettyRemotingServer;
+import org.apache.dolphinscheduler.remote.command.CommandType;
+import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
+import org.apache.dolphinscheduler.server.log.LoggerRequestProcessor;
+import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.processor.HostUpdateProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteResponseAckProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskExecuteRunningAckProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskKillProcessor;
+import org.apache.dolphinscheduler.server.worker.processor.TaskRecallAckProcessor;
+
+import java.io.Closeable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Service
+public class WorkerRpcServer implements Closeable {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(WorkerRpcServer.class);
+
+    @Autowired
+    private TaskExecuteProcessor taskExecuteProcessor;
+
+    @Autowired
+    private TaskKillProcessor taskKillProcessor;
+
+    @Autowired
+    private TaskRecallAckProcessor taskRecallAckProcessor;
+
+    @Autowired
+    private TaskExecuteRunningAckProcessor taskExecuteRunningAckProcessor;
+
+    @Autowired
+    private TaskExecuteResponseAckProcessor taskExecuteResponseAckProcessor;
+
+    @Autowired
+    private HostUpdateProcessor hostUpdateProcessor;
+
+    @Autowired
+    private LoggerRequestProcessor loggerRequestProcessor;
+
+    @Autowired
+    private WorkerConfig workerConfig;
+
+    private NettyRemotingServer nettyRemotingServer;
+
+    public void start() {
+        LOGGER.info("Worker rpc server starting");
+        NettyServerConfig serverConfig = new NettyServerConfig();
+        serverConfig.setListenPort(workerConfig.getListenPort());
+        this.nettyRemotingServer = new NettyRemotingServer(serverConfig);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_REQUEST, taskExecuteProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_KILL_REQUEST, taskKillProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RUNNING_ACK, taskExecuteRunningAckProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_EXECUTE_RESPONSE_ACK, taskExecuteResponseAckProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.PROCESS_HOST_UPDATE_REQUEST, hostUpdateProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.TASK_RECALL_ACK, taskRecallAckProcessor);
+        // logger server
+        this.nettyRemotingServer.registerProcessor(CommandType.GET_LOG_BYTES_REQUEST, loggerRequestProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.ROLL_VIEW_LOG_REQUEST, loggerRequestProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.VIEW_WHOLE_LOG_REQUEST, loggerRequestProcessor);
+        this.nettyRemotingServer.registerProcessor(CommandType.REMOVE_TAK_LOG_REQUEST, loggerRequestProcessor);
+        this.nettyRemotingServer.start();
+        LOGGER.info("Worker rpc server started");
+    }
+
+    @Override
+    public void close() {
+        LOGGER.info("Worker rpc server closing");
+        this.nettyRemotingServer.close();
+        LOGGER.info("Worker rpc server closed");
+    }
+
+}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
index 34382257d7..b355c6702b 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
@@ -18,10 +18,12 @@
 package org.apache.dolphinscheduler.server.worker.processor;
 
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.storage.StorageOperate;
 import org.apache.dolphinscheduler.common.utils.CommonUtils;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.common.utils.NetUtils;
 import org.apache.dolphinscheduler.common.utils.OSUtils;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -90,15 +92,18 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
     @Autowired
     private WorkerManagerThread workerManager;
 
-    @Counted(value = "dolphinscheduler_task_execution_count", description = "task execute total count")
-    @Timed(value = "dolphinscheduler_task_execution_timer", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
+    @Autowired(required = false)
+    private StorageOperate storageOperate;
+
+    @Counted(value = "ds.task.execution.count", description = "task execute total count")
+    @Timed(value = "ds.task.execution.duration", percentiles = {0.5, 0.75, 0.95, 0.99}, histogram = true)
     @Override
     public void process(Channel channel, Command command) {
         Preconditions.checkArgument(CommandType.TASK_EXECUTE_REQUEST == command.getType(),
-                String.format("invalid command type : %s", command.getType()));
+                                    String.format("invalid command type : %s", command.getType()));
 
-        TaskExecuteRequestCommand taskRequestCommand = JSONUtils.parseObject(
-                command.getBody(), TaskExecuteRequestCommand.class);
+        TaskExecuteRequestCommand taskRequestCommand = JSONUtils.parseObject(command.getBody(),
+                                                                             TaskExecuteRequestCommand.class);
 
         if (taskRequestCommand == null) {
             logger.error("task execute request command is null");
@@ -113,70 +118,98 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
             logger.error("task execution context is null");
             return;
         }
-        TaskMetrics.incrTaskTypeExecuteCount(taskExecutionContext.getTaskType());
-
-        // set cache, it will be used when kill task
-        TaskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
-
-        // todo custom logger
-
-        taskExecutionContext.setHost(NetUtils.getAddr(workerConfig.getListenPort()));
-        taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext));
-
-        if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) {
-            if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) {
-                OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode());
-            }
-
-            // check if the OS user exists
-            if (!OSUtils.getUserList().contains(taskExecutionContext.getTenantCode())) {
-                logger.error("tenantCode: {} does not exist, taskInstanceId: {}",
-                        taskExecutionContext.getTenantCode(), taskExecutionContext.getTaskInstanceId());
-                TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-                taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
-                taskExecutionContext.setEndTime(new Date());
-                taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
-                return;
-            }
-
-            // local execute path
-            String execLocalPath = getExecLocalPath(taskExecutionContext);
-            logger.info("task instance local execute path : {}", execLocalPath);
-            taskExecutionContext.setExecutePath(execLocalPath);
-
-            try {
-                FileUtils.createWorkDirIfAbsent(execLocalPath);
-            } catch (Throwable ex) {
-                logger.error("create execLocalPath fail, path: {}, taskInstanceId: {}", execLocalPath, taskExecutionContext.getTaskInstanceId());
-                logger.error("create executeLocalPath fail", ex);
-                TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
-                taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
-                taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
-                return;
+        try {
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(),
+                                                        taskExecutionContext.getTaskInstanceId());
+
+            TaskMetrics.incrTaskTypeExecuteCount(taskExecutionContext.getTaskType());
+
+            // set cache, it will be used when kill task
+            TaskExecutionContextCacheManager.cacheTaskExecutionContext(taskExecutionContext);
+
+            // todo custom logger
+
+            taskExecutionContext.setHost(NetUtils.getAddr(workerConfig.getListenPort()));
+            taskExecutionContext.setLogPath(LogUtils.getTaskLogPath(taskExecutionContext));
+
+            if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) {
+                boolean osUserExistFlag;
+                //if Using distributed is true and Currently supported systems are linux,Should not let it automatically
+                //create tenants,so TenantAutoCreate has no effect
+                if (workerConfig.isTenantDistributedUser() && SystemUtils.IS_OS_LINUX) {
+                    //use the id command to judge in linux
+                    osUserExistFlag = OSUtils.existTenantCodeInLinux(taskExecutionContext.getTenantCode());
+                } else if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) {
+                    // if not exists this user, then create
+                    OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode());
+                    osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode());
+                } else {
+                    osUserExistFlag = OSUtils.getUserList().contains(taskExecutionContext.getTenantCode());
+                }
+                if (Constants.DRY_RUN_FLAG_NO == taskExecutionContext.getDryRun()) {
+                    if (CommonUtils.isSudoEnable() && workerConfig.isTenantAutoCreate()) {
+                        OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode());
+                    }
+
+                    // check if the OS user exists
+                    if (!OSUtils.getUserList().contains(taskExecutionContext.getTenantCode())) {
+                        logger.error("tenantCode: {} does not exist, taskInstanceId: {}",
+                                     taskExecutionContext.getTenantCode(),
+                                     taskExecutionContext.getTaskInstanceId());
+                        TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+                        taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
+                        taskExecutionContext.setEndTime(new Date());
+                        taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                        return;
+                    }
+
+                    // local execute path
+                    String execLocalPath = getExecLocalPath(taskExecutionContext);
+                    logger.info("task instance local execute path : {}", execLocalPath);
+                    taskExecutionContext.setExecutePath(execLocalPath);
+
+                    try {
+                        FileUtils.createWorkDirIfAbsent(execLocalPath);
+                    } catch (Throwable ex) {
+                        logger.error("create execLocalPath fail, path: {}, taskInstanceId: {}",
+                                     execLocalPath,
+                                     taskExecutionContext.getTaskInstanceId());
+                        logger.error("create executeLocalPath fail", ex);
+                        TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
+                        taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
+                        taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                        return;
+                    }
+                }
+
+                taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
+                                                     new NettyRemoteChannel(channel, command.getOpaque()));
+
+                // delay task process
+                long remainTime = DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(),
+                                                          taskExecutionContext.getDelayTime() * 60L);
+                if (remainTime > 0) {
+                    logger.info("delay the execution of task instance {}, delay time: {} s",
+                                taskExecutionContext.getTaskInstanceId(),
+                                remainTime);
+                    taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION);
+                    taskExecutionContext.setStartTime(null);
+                    taskCallbackService.sendTaskExecuteDelayCommand(taskExecutionContext);
+                }
+
+                // submit task to manager
+                boolean offer = workerManager.offer(new TaskExecuteThread(taskExecutionContext,
+                                                                          taskCallbackService,
+                                                                          alertClientService,
+                                                                          taskPluginManager));
+                if (!offer) {
+                    logger.error("submit task to manager error, queue is full, queue size is {}, taskInstanceId: {}",
+                                 workerManager.getDelayQueueSize(),
+                                 taskExecutionContext.getTaskInstanceId());
+                    taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
+                    taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
+                }
             }
-        }
-
-        taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
-                new NettyRemoteChannel(channel, command.getOpaque()));
-
-        // delay task process
-        long remainTime = DateUtils.getRemainTime(taskExecutionContext.getFirstSubmitTime(), taskExecutionContext.getDelayTime() * 60L);
-        if (remainTime > 0) {
-            logger.info("delay the execution of task instance {}, delay time: {} s", taskExecutionContext.getTaskInstanceId(), remainTime);
-            taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.DELAY_EXECUTION);
-            taskExecutionContext.setStartTime(null);
-            taskCallbackService.sendTaskExecuteDelayCommand(taskExecutionContext);
-        }
-
-        // submit task to manager
-        boolean offer = workerManager.offer(new TaskExecuteThread(taskExecutionContext, taskCallbackService, alertClientService, taskPluginManager));
-        if (!offer) {
-            logger.error("submit task to manager error, queue is full, queue size is {}, taskInstanceId: {}",
-                    workerManager.getDelayQueueSize(), taskExecutionContext.getTaskInstanceId());
-            taskExecutionContext.setCurrentExecutionStatus(ExecutionStatus.FAILURE);
-            taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
-        }
-    }
 
     /**
      * get execute local path
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java
index 9d74dc5dcc..421e92a530 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteRunningAckProcessor.java
@@ -18,6 +18,7 @@
 package org.apache.dolphinscheduler.server.worker.processor;
 
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.remote.command.Command;
 import org.apache.dolphinscheduler.remote.command.CommandType;
@@ -44,19 +45,23 @@ public class TaskExecuteRunningAckProcessor implements NettyRequestProcessor {
     @Override
     public void process(Channel channel, Command command) {
         Preconditions.checkArgument(CommandType.TASK_EXECUTE_RUNNING_ACK == command.getType(),
-                String.format("invalid command type : %s", command.getType()));
+            String.format("invalid command type : %s", command.getType()));
 
         TaskExecuteRunningAckCommand runningAckCommand = JSONUtils.parseObject(
-                command.getBody(), TaskExecuteRunningAckCommand.class);
-
+            command.getBody(), TaskExecuteRunningAckCommand.class);
         if (runningAckCommand == null) {
             logger.error("task execute running ack command is null");
             return;
         }
-        logger.info("task execute running ack command : {}", runningAckCommand);
+        try {
+            LoggerUtils.setTaskInstanceIdMDC(runningAckCommand.getTaskInstanceId());
+            logger.info("task execute running ack command : {}", runningAckCommand);
 
-        if (runningAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()) {
-            ResponseCache.get().removeRunningCache(runningAckCommand.getTaskInstanceId());
+            if (runningAckCommand.getStatus() == ExecutionStatus.SUCCESS.getCode()) {
+                ResponseCache.get().removeRunningCache(runningAckCommand.getTaskInstanceId());
+            }
+        } finally {
+            LoggerUtils.removeTaskInstanceIdMDC();
         }
     }
 
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
index fc737ca1de..0e48465a07 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/RetryReportTaskStatusThread.java
@@ -47,9 +47,11 @@ public class RetryReportTaskStatusThread implements Runnable {
     private TaskCallbackService taskCallbackService;
 
     public void start() {
+        logger.info("Retry report task status thread starting");
         Thread thread = new Thread(this, "RetryReportTaskStatusThread");
         thread.setDaemon(true);
         thread.start();
+        logger.info("Retry report task status thread started");
     }
 
     /**
@@ -83,7 +85,7 @@ public class RetryReportTaskStatusThread implements Runnable {
                     }
                 }
             } catch (Exception e) {
-                logger.warn("retry report task status error", e);
+                logger.warn("Retry report task status error", e);
             }
         }
     }
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
index e6ff2b5653..ae89e9e946 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@@ -76,14 +76,6 @@ public class TaskExecuteThread implements Runnable, Delayed {
      */
     private TaskExecutionContext taskExecutionContext;
 
-    public StorageOperate getStorageOperate() {
-        return storageOperate;
-    }
-
-    public void setStorageOperate(StorageOperate storageOperate) {
-        this.storageOperate = storageOperate;
-    }
-
     private StorageOperate storageOperate;
 
     /**
@@ -107,24 +99,28 @@ public class TaskExecuteThread implements Runnable, Delayed {
      * constructor
      *
      * @param taskExecutionContext taskExecutionContext
-     * @param taskCallbackService taskCallbackService
+     * @param taskCallbackService  taskCallbackService
      */
     public TaskExecuteThread(TaskExecutionContext taskExecutionContext,
                              TaskCallbackService taskCallbackService,
-                             AlertClientService alertClientService) {
+                             AlertClientService alertClientService,
+                             StorageOperate storageOperate) {
         this.taskExecutionContext = taskExecutionContext;
         this.taskCallbackService = taskCallbackService;
         this.alertClientService = alertClientService;
+        this.storageOperate = storageOperate;
     }
 
     public TaskExecuteThread(TaskExecutionContext taskExecutionContext,
                              TaskCallbackService taskCallbackService,
                              AlertClientService alertClientService,
-                             TaskPluginManager taskPluginManager) {
+                             TaskPluginManager taskPluginManager,
+                             StorageOperate storageOperate) {
         this.taskExecutionContext = taskExecutionContext;
         this.taskCallbackService = taskCallbackService;
         this.alertClientService = alertClientService;
         this.taskPluginManager = taskPluginManager;
+        this.storageOperate = storageOperate;
     }
 
     @Override
@@ -139,6 +135,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
         }
 
         try {
+            LoggerUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId());
             logger.info("script path : {}", taskExecutionContext.getExecutePath());
             if (taskExecutionContext.getStartTime() == null) {
                 taskExecutionContext.setStartTime(new Date());
@@ -151,7 +148,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
 
             // copy hdfs/minio file to local
             List<Pair<String, String>> fileDownloads = downloadCheck(taskExecutionContext.getExecutePath(), taskExecutionContext.getResources());
-            if (!fileDownloads.isEmpty()){
+            if (!fileDownloads.isEmpty()) {
                 downloadResource(taskExecutionContext.getExecutePath(), logger, fileDownloads);
             }
 
@@ -211,6 +208,7 @@ public class TaskExecuteThread implements Runnable, Delayed {
             TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
             taskCallbackService.sendTaskExecuteResponseCommand(taskExecutionContext);
             clearTaskExecPath();
+            LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
         }
     }
 
@@ -308,11 +306,12 @@ public class TaskExecuteThread implements Runnable, Delayed {
 
     /**
      * download resource check
+     *
      * @param execLocalPath
      * @param projectRes
      * @return
      */
-    public List<Pair<String, String>> downloadCheck(String execLocalPath, Map<String, String> projectRes){
+    public List<Pair<String, String>> downloadCheck(String execLocalPath, Map<String, String> projectRes) {
         if (MapUtils.isEmpty(projectRes)) {
             return Collections.emptyList();
         }
@@ -320,13 +319,13 @@ public class TaskExecuteThread implements Runnable, Delayed {
         projectRes.forEach((key, value) -> {
             File resFile = new File(execLocalPath, key);
             boolean notExist = !resFile.exists();
-            if (notExist){
+            if (notExist) {
                 downloadFile.add(Pair.of(key, value));
-            } else{
+            } else {
                 logger.info("file : {} exists ", resFile.getName());
             }
         });
-        if (!downloadFile.isEmpty() && !PropertyUtils.getResUploadStartupState()){
+        if (!downloadFile.isEmpty() && !PropertyUtils.getResUploadStartupState()) {
             throw new StorageOperateNoConfiguredException("Storage service config does not exist!");
         }
         return downloadFile;
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
index 2702d4af42..ae735f4338 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerManagerThread.java
@@ -24,13 +24,10 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager;
 import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
 import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
-import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics;
 import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
 
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.DelayQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -132,9 +129,11 @@ public class WorkerManagerThread implements Runnable {
     }
 
     public void start() {
+        logger.info("Worker manager thread starting");
         Thread thread = new Thread(this, this.getClass().getName());
         thread.setDaemon(true);
         thread.start();
+        logger.info("Worker manager thread started");
     }
 
     @Override
diff --git a/dolphinscheduler-worker/src/main/resources/logback-spring.xml b/dolphinscheduler-worker/src/main/resources/logback-spring.xml
index c6d8ee415d..571a3addd2 100644
--- a/dolphinscheduler-worker/src/main/resources/logback-spring.xml
+++ b/dolphinscheduler-worker/src/main/resources/logback-spring.xml
@@ -22,7 +22,7 @@
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <pattern>
-                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n
             </pattern>
             <charset>UTF-8</charset>
         </encoder>
@@ -58,7 +58,7 @@
         </rollingPolicy>
         <encoder>
             <pattern>
-                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
+                [%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n
             </pattern>
             <charset>UTF-8</charset>
         </encoder>
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
index 2bb23cad6a..13c533f1cf 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.dolphinscheduler.server.worker.processor;
 
+import org.apache.dolphinscheduler.common.storage.StorageOperate;
 import org.apache.dolphinscheduler.common.thread.ThreadUtils;
 import org.apache.dolphinscheduler.common.utils.FileUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
@@ -53,7 +54,7 @@ import org.slf4j.Logger;
  */
 @RunWith(PowerMockRunner.class)
 @PrepareForTest({SpringApplicationContext.class, TaskCallbackService.class, WorkerConfig.class, FileUtils.class,
-        JsonSerializer.class, JSONUtils.class, ThreadUtils.class, ExecutorService.class, ChannelUtils.class})
+    JsonSerializer.class, JSONUtils.class, ThreadUtils.class, ExecutorService.class, ChannelUtils.class})
 @Ignore
 public class TaskExecuteProcessorTest {
 
@@ -63,6 +64,8 @@ public class TaskExecuteProcessorTest {
 
     private ExecutorService workerExecService;
 
+    private StorageOperate storageOperate;
+
     private WorkerConfig workerConfig;
 
     private Command command;
@@ -99,19 +102,23 @@ public class TaskExecuteProcessorTest {
 
         PowerMockito.mockStatic(SpringApplicationContext.class);
         PowerMockito.when(SpringApplicationContext.getBean(TaskCallbackService.class))
-                .thenReturn(taskCallbackService);
+            .thenReturn(taskCallbackService);
         PowerMockito.when(SpringApplicationContext.getBean(WorkerConfig.class))
-                .thenReturn(workerConfig);
+            .thenReturn(workerConfig);
 
         workerManager = PowerMockito.mock(WorkerManagerThread.class);
-        PowerMockito.when(workerManager.offer(new TaskExecuteThread(taskExecutionContext, taskCallbackService, alertClientService))).thenReturn(Boolean.TRUE);
+
+        storageOperate = PowerMockito.mock(StorageOperate.class);
+        PowerMockito.when(
+                workerManager.offer(new TaskExecuteThread(taskExecutionContext, taskCallbackService, alertClientService, storageOperate)))
+            .thenReturn(Boolean.TRUE);
 
         PowerMockito.when(SpringApplicationContext.getBean(WorkerManagerThread.class))
-                .thenReturn(workerManager);
+            .thenReturn(workerManager);
 
         PowerMockito.mockStatic(ThreadUtils.class);
         PowerMockito.when(ThreadUtils.newDaemonFixedThreadExecutor("Worker-Execute-Thread", workerConfig.getExecThreads()))
-                .thenReturn(workerExecService);
+            .thenReturn(workerExecService);
 
         PowerMockito.mockStatic(JsonSerializer.class);
         PowerMockito.when(JsonSerializer.deserialize(command.getBody(), TaskExecuteRequestCommand.class))
@@ -125,16 +132,17 @@ public class TaskExecuteProcessorTest {
 
         PowerMockito.mockStatic(FileUtils.class);
         PowerMockito.when(FileUtils.getProcessExecDir(taskExecutionContext.getProjectCode(),
-                        taskExecutionContext.getProcessDefineCode(),
-                        taskExecutionContext.getProcessDefineVersion(),
-                        taskExecutionContext.getProcessInstanceId(),
-                        taskExecutionContext.getTaskInstanceId()))
-                .thenReturn(taskExecutionContext.getExecutePath());
+                taskExecutionContext.getProcessDefineCode(),
+                taskExecutionContext.getProcessDefineVersion(),
+                taskExecutionContext.getProcessInstanceId(),
+                taskExecutionContext.getTaskInstanceId()))
+            .thenReturn(taskExecutionContext.getExecutePath());
         PowerMockito.doNothing().when(FileUtils.class, "createWorkDirIfAbsent", taskExecutionContext.getExecutePath());
 
-        SimpleTaskExecuteThread simpleTaskExecuteThread = new SimpleTaskExecuteThread(null, null, null, alertClientService);
+        SimpleTaskExecuteThread simpleTaskExecuteThread = new SimpleTaskExecuteThread(
+            null, null, null, alertClientService, storageOperate);
         PowerMockito.whenNew(TaskExecuteThread.class).withAnyArguments()
-                .thenReturn(simpleTaskExecuteThread);
+            .thenReturn(simpleTaskExecuteThread);
     }
 
     @Test
@@ -172,8 +180,12 @@ public class TaskExecuteProcessorTest {
 
     private static class SimpleTaskExecuteThread extends TaskExecuteThread {
 
-        public SimpleTaskExecuteThread(TaskExecutionContext taskExecutionContext, TaskCallbackService taskCallbackService, Logger taskLogger, AlertClientService alertClientService) {
-            super(taskExecutionContext, taskCallbackService, alertClientService);
+        public SimpleTaskExecuteThread(TaskExecutionContext taskExecutionContext,
+                                       TaskCallbackService taskCallbackService,
+                                       Logger taskLogger,
+                                       AlertClientService alertClientService,
+                                       StorageOperate storageOperate) {
+            super(taskExecutionContext, taskCallbackService, alertClientService, storageOperate);
         }
 
         @Override
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java
index f847690a6c..90577111b9 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThreadTest.java
@@ -17,12 +17,20 @@
 
 package org.apache.dolphinscheduler.server.worker.runner;
 
-import org.apache.commons.lang3.tuple.Pair;
+import org.apache.dolphinscheduler.common.storage.StorageOperate;
 import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
 import org.apache.dolphinscheduler.server.worker.processor.TaskCallbackService;
 import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClientTest;
 import org.apache.dolphinscheduler.service.alert.AlertClientService;
 import org.apache.dolphinscheduler.service.task.TaskPluginManager;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -31,11 +39,6 @@ import org.powermock.modules.junit4.PowerMockRunner;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 @RunWith(PowerMockRunner.class)
 public class TaskExecuteThreadTest {
 
@@ -50,20 +53,24 @@ public class TaskExecuteThreadTest {
     @Mock
     private AlertClientService alertClientService;
 
+    @Mock
+    private StorageOperate storageOperate;
+
     @Mock
     private TaskPluginManager taskPluginManager;
 
     @Test
-    public void checkTest(){
-        TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService, alertClientService, taskPluginManager);
+    public void checkTest() {
+        TaskExecuteThread taskExecuteThread = new TaskExecuteThread(taskExecutionContext, taskCallbackService,
+            alertClientService, taskPluginManager, storageOperate);
 
         String path = "/";
         Map<String, String> projectRes = new HashMap<>();
         projectRes.put("shell", "shell.sh");
         List<Pair<String, String>> downloads = new ArrayList<>();
-        try{
+        try {
             downloads = taskExecuteThread.downloadCheck(path, projectRes);
-        }catch (Exception e){
+        } catch (Exception e) {
             Assert.assertNotNull(e);
         }
         downloads.add(Pair.of("shell", "shell.sh"));


[dolphinscheduler] 20/29: [Fix-10785] Fix state event handle error will not retry (#10786)

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

wenjun pushed a commit to branch 3.0.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git

commit 4fc9bce444b482248a68b0cf2a310c59d848ef95
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Wed Jul 6 14:53:28 2022 +0800

    [Fix-10785] Fix state event handle error will not retry (#10786)
    
    * Fix state event handle error will not retry
    
    * Use state event handler to deal with the event
    
    (cherry picked from commit 67d14fb7b3d941af613a96a0b9c3a2928e5c201c)
---
 .../master/runner/WorkflowExecuteRunnable.java     | 35 ++++++++++------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 7a15a43d2b..16604662ac 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -281,14 +281,14 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             try {
                 stateEvent = this.stateEvents.peek();
                 LoggerUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(),
-                                                            stateEvent.getTaskInstanceId());
+                    stateEvent.getTaskInstanceId());
                 // if state handle success then will remove this state, otherwise will retry this state next time.
                 // The state should always handle success except database error.
                 checkProcessInstance(stateEvent);
 
-                StateEventHandler stateEventHandler
-                    = StateEventHandlerManager.getStateEventHandler(stateEvent.getType())
-                    .orElseThrow(() -> new StateEventHandleError("Cannot find handler for the given state event"));
+                StateEventHandler stateEventHandler =
+                    StateEventHandlerManager.getStateEventHandler(stateEvent.getType())
+                        .orElseThrow(() -> new StateEventHandleError("Cannot find handler for the given state event"));
                 if (stateEventHandler.handleStateEvent(this, stateEvent)) {
                     this.stateEvents.remove(stateEvent);
                 }
@@ -298,14 +298,14 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
                 ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
             } catch (StateEventHandleException stateEventHandleException) {
                 logger.error("State event handle error, will retry this event: {}",
-                             stateEvent,
-                             stateEventHandleException);
+                    stateEvent,
+                    stateEventHandleException);
                 ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
             } catch (Exception e) {
                 // we catch the exception here, since if the state event handle failed, the state event will still keep in the stateEvents queue.
                 logger.error("State event handle error, get a unknown exception, will retry this event: {}",
-                             stateEvent,
-                             e);
+                    stateEvent,
+                    e);
                 ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
             } finally {
                 LoggerUtils.removeWorkflowAndTaskInstanceIdMDC();
@@ -350,7 +350,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
             ITaskProcessor taskProcessor = activeTaskProcessorMaps.get(taskInstance.getTaskCode());
             taskProcessor.action(TaskAction.DISPATCH);
             this.processService.updateTaskGroupQueueStatus(taskGroupQueue.getTaskId(),
-                                                           TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode());
+                TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode());
             return true;
         }
         if (taskGroupQueue.getInQueue() == Flag.YES.getCode()) {
@@ -454,8 +454,8 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         TaskInstance newTaskInstance = cloneRetryTaskInstance(taskInstance);
         if (newTaskInstance == null) {
             logger.error("retry fail, new taskInstance is null, task code:{}, task id:{}",
-                         taskInstance.getTaskCode(),
-                         taskInstance.getId());
+                taskInstance.getTaskCode(),
+                taskInstance.getId());
             return;
         }
         waitToRetryTaskInstanceMap.put(newTaskInstance.getTaskCode(), newTaskInstance);
@@ -787,8 +787,8 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         errorTaskMap.clear();
 
         if (!isNewProcessInstance()) {
-            List<TaskInstance> validTaskInstanceList
-                = processService.findValidTaskListByProcessId(processInstance.getId());
+            List<TaskInstance> validTaskInstanceList =
+                processService.findValidTaskListByProcessId(processInstance.getId());
             for (TaskInstance task : validTaskInstanceList) {
                 if (validTaskMap.containsKey(task.getTaskCode())) {
                     int oldTaskInstanceId = validTaskMap.get(task.getTaskCode());
@@ -799,7 +799,7 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
                         continue;
                     }
                     logger.warn("have same taskCode taskInstance when init task queue, taskCode:{}",
-                                task.getTaskCode());
+                        task.getTaskCode());
                 }
 
                 validTaskMap.put(task.getTaskCode(), task.getId());
@@ -1193,10 +1193,8 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
     }
 
     private void submitPostNode(String parentNodeCode) throws StateEventHandleException {
-        Set<String> submitTaskNodeList = DagHelper.parsePostNodes(parentNodeCode,
-                                                                  skipTaskNodeMap,
-                                                                  dag,
-                                                                  getCompleteTaskInstanceMap());
+        Set<String> submitTaskNodeList =
+            DagHelper.parsePostNodes(parentNodeCode, skipTaskNodeMap, dag, getCompleteTaskInstanceMap());
         List<TaskInstance> taskInstances = new ArrayList<>();
         for (String taskNode : submitTaskNodeList) {
             TaskNode taskNodeObject = dag.getNode(taskNode);
@@ -1859,7 +1857,6 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
         return waitToRetryTaskInstanceMap;
     }
 
-
     private void setGlobalParamIfCommanded(ProcessDefinition processDefinition, Map<String, String> cmdParam) {
         // get start params from command param
         Map<String, String> startParamMap = new HashMap<>();