You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zi...@apache.org on 2024/11/29 07:26:00 UTC
(dolphinscheduler) branch dev updated: [DSIP-73] Add dolphinscheduler-task-executor module to unify the task execution logic (#16790)
This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 673687d540 [DSIP-73] Add dolphinscheduler-task-executor module to unify the task execution logic (#16790)
673687d540 is described below
commit 673687d540935da1b52c886c83b06045c8b07f41
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Fri Nov 29 15:25:54 2024 +0800
[DSIP-73] Add dolphinscheduler-task-executor module to unify the task execution logic (#16790)
---
docs/docs/en/architecture/configuration.md | 41 +--
docs/docs/en/guide/upgrade/incompatible.md | 1 +
docs/docs/zh/architecture/configuration.md | 34 +-
docs/docs/zh/guide/upgrade/incompatible.md | 1 +
.../api/service/impl/TaskInstanceServiceImpl.java | 14 +-
.../api/service/LoggerServiceTest.java | 7 -
.../common/thread/ThreadUtils.java | 11 +-
.../eventbus/AbstractDelayEvent.java | 32 +-
.../eventbus/AbstractDelayEventBus.java | 10 +
.../dolphinscheduler/eventbus/IEventBus.java | 10 +
.../dolphinscheduler-extract-base/pom.xml | 5 +-
.../extract/base/serialize/JsonSerializer.java | 46 +--
.../extract/base/server/NettyRemotingServer.java | 6 +-
.../dolphinscheduler-extract-common/pom.xml | 2 -
.../extract/common/ILogService.java | 5 -
.../dolphinscheduler-extract-master/pom.xml | 12 +-
.../extract/master/ILogicTaskExecutorOperator.java | 45 +++
.../extract/master/ILogicTaskInstanceOperator.java | 41 ---
.../master/ITaskExecutionEventListener.java | 50 ---
.../extract/master/ITaskExecutorEventListener.java | 54 +++
.../TaskExecutorEventRemoteReporterClient.java | 120 +++++++
.../master/transportor/ITaskExecutionEvent.java | 7 +-
.../transportor/StreamingTaskTriggerRequest.java | 51 ---
.../transportor/TaskExecutionSuccessEvent.java | 55 ---
.../dolphinscheduler-extract-worker/pom.xml | 8 +-
.../worker/IPhysicalTaskExecutorOperator.java | 50 +++
.../ITaskInstanceExecutionEventAckListener.java | 51 ---
.../extract/worker/ITaskInstanceOperator.java | 46 ---
.../transportor/TaskExecutionFailedEventAck.java | 40 ---
.../transportor/TaskExecutionKilledEventAck.java | 40 ---
.../transportor/TaskExecutionPausedEventAck.java | 40 ---
.../transportor/TaskExecutionSuccessEventAck.java | 40 ---
.../transportor/TaskInstanceDispatchResponse.java | 42 ---
.../TaskInstanceExecutionDispatchedEventAck.java | 39 ---
.../TaskInstanceExecutionFinishEventAck.java | 40 ---
.../TaskInstanceExecutionRunningEventAck.java | 40 ---
.../transportor/TaskInstanceKillResponse.java | 74 ----
.../TaskInstanceTriggerSavepointRequest.java | 1 +
.../TaskInstanceTriggerSavepointResponse.java | 1 +
dolphinscheduler-master/pom.xml | 6 +-
.../server/master/MasterServer.java | 4 +-
.../server/master/config/LogicTaskConfig.java | 12 +-
.../server/master/config/MasterConfig.java | 5 +-
.../server/master/engine/IWorkflowRepository.java | 1 -
.../server/master/engine/TaskGroupCoordinator.java | 8 +-
.../master/engine/WorkflowCacheRepository.java | 4 -
.../server/master/engine/WorkflowEngine.java | 26 +-
.../master/engine/WorkflowEventBusFireWorker.java | 6 +-
.../master/engine/WorkflowEventBusFireWorkers.java | 9 +-
.../engine/command/IdSlotBasedCommandFetcher.java | 2 +-
.../TaskReassignMasterHostException.java | 17 +-
.../engine/executor/LogicTaskEngineDelegator.java | 78 +++++
.../engine/executor/LogicTaskEngineFactory.java | 52 +++
.../master/engine/executor/LogicTaskExecutor.java | 88 +++++
.../engine/executor/LogicTaskExecutorBuilder.java | 29 +-
.../LogicTaskExecutorContainerProvider.java | 45 +++
.../LogicTaskExecutorEventBusCoordinator.java} | 28 +-
...LogicTaskExecutorEventRemoteReporterClient.java | 9 +-
.../engine/executor/LogicTaskExecutorFactory.java | 55 +++
.../LogicTaskExecutorLifecycleEventListener.java} | 37 +-
.../LogicTaskExecutorLifecycleEventReporter.java | 24 +-
.../executor/LogicTaskExecutorRepository.java | 9 +-
.../engine/executor/plugin/AbstractLogicTask.java | 74 ++++
.../executor/plugin}/ILogicTask.java | 12 +-
.../executor/plugin}/ILogicTaskPluginFactory.java | 9 +-
.../plugin/ITaskParameterDeserializer.java | 6 +-
.../plugin}/LogicTaskPluginFactoryBuilder.java | 14 +-
.../plugin}/condition/ConditionLogicTask.java | 47 ++-
.../condition/ConditionLogicTaskPluginFactory.java | 10 +-
.../plugin}/dependent/DependentLogicTask.java | 68 ++--
.../dependent/DependentLogicTaskPluginFactory.java | 11 +-
.../plugin/dependent/DependentTaskTracker.java} | 37 +-
.../dynamic/DynamicAsyncTaskExecuteFunction.java | 24 +-
.../plugin}/dynamic/DynamicCommandUtils.java | 2 +-
.../executor/plugin}/dynamic/DynamicLogicTask.java | 75 +++--
.../dynamic/DynamicLogicTaskPluginFactory.java | 16 +-
.../executor/plugin}/dynamic/DynamicOutput.java | 2 +-
.../executor/plugin}/fake/LogicFakeTask.java | 55 +--
.../plugin}/fake/LogicFakeTaskPluginFactory.java | 25 +-
.../subworkflow/SubWorkflowControlClient.java | 4 +-
.../plugin}/subworkflow/SubWorkflowLogicTask.java | 49 ++-
.../SubWorkflowLogicTaskPluginFactory.java | 34 +-
.../SubWorkflowLogicTaskRuntimeContext.java | 2 +-
.../plugin/subworkflow/SubWorkflowTracker.java} | 33 +-
.../trigger/SubWorkflowManualTrigger.java | 2 +-
.../plugin}/switchtask/SwitchLogicTask.java | 40 ++-
.../switchtask/SwitchLogicTaskPluginFactory.java | 11 +-
.../master/engine/graph/SuccessorFlowAdjuster.java | 4 +-
.../master/engine/system/SystemEventBus.java | 20 +-
.../engine/system/event/AbstractSystemEvent.java | 4 +
.../system/event/GlobalMasterFailoverEvent.java | 2 +-
.../engine/task/client/ITaskExecutorClient.java | 33 +-
.../task/client/ITaskExecutorClientDelegator.java | 27 +-
.../master/engine/task/client/ITaskOperator.java | 31 --
.../client/LogicTaskExecutorClientDelegator.java | 69 +++-
.../PhysicalTaskExecutorClientDelegator.java | 120 ++++++-
.../engine/task/client/TaskExecutorClient.java | 53 ++-
.../task/lifecycle/TaskLifecycleEventType.java | 4 +
.../lifecycle/event/TaskRunningLifecycleEvent.java | 2 -
...nt.java => TaskRuntimeContextChangedEvent.java} | 13 +-
.../handler/AbstractTaskLifecycleEventHandler.java | 2 +-
.../TaskDispatchedLifecycleEventHandler.java | 14 +
.../handler/TaskFailedLifecycleEventHandler.java | 14 +
.../handler/TaskKilledLifecycleEventHandler.java | 14 +
.../handler/TaskPausedLifecycleEventHandler.java | 14 +
.../handler/TaskRunningLifecycleEventHandler.java | 14 +
...untimeContextChangedLifecycleEventHandler.java} | 28 +-
.../handler/TaskSuccessLifecycleEventHandler.java | 14 +
.../handler/TaskTimeoutLifecycleEventHandler.java | 8 +-
.../task/runnable/ITaskExecutionRunnable.java | 8 +
.../task/runnable/TaskExecutionRunnable.java | 29 +-
.../runnable/TaskExecutionRunnableBuilder.java | 4 +-
.../task/statemachine/AbstractTaskStateAction.java | 15 +-
.../engine/task/statemachine/ITaskStateAction.java | 9 +
.../task/statemachine/TaskDispatchStateAction.java | 2 +-
.../statemachine/TaskSubmittedStateAction.java | 2 +-
.../statemachine/AbstractWorkflowStateAction.java | 1 -
...ogicITaskInstanceDispatchOperationFunction.java | 81 -----
.../LogicITaskInstanceKillOperationFunction.java | 65 ----
.../LogicITaskInstancePauseOperationFunction.java | 62 ----
.../master/rpc/LogicTaskExecutorOperatorImpl.java | 105 ++++++
.../LogicTaskInstanceOperationFunctionManager.java | 47 ---
.../master/rpc/LogicTaskInstanceOperatorImpl.java | 58 ----
.../server/master/rpc/MasterLogServiceImpl.java | 8 -
.../master/rpc/TaskExecutionEventListenerImpl.java | 210 ------------
.../master/rpc/TaskExecutorEventListenerImpl.java | 148 ++++++++
.../GlobalTaskDispatchWaitingQueueLooper.java | 8 +-
.../master/runner/MasterTaskExecutorBootstrap.java | 56 ----
.../master/runner/TaskExecuteRunnableFactory.java | 44 ---
.../runner/dispatcher/BaseTaskDispatcher.java | 63 ----
.../runner/dispatcher/MasterTaskDispatcher.java | 74 ----
.../master/runner/dispatcher/TaskDispatcher.java | 30 --
.../runner/dispatcher/WorkerTaskDispatcher.java | 74 ----
.../execute/AsyncMasterTaskDelayQueueLooper.java | 133 --------
.../runner/execute/AsyncMasterTaskExecutor.java | 60 ----
.../execute/AsyncMasterTaskExecutorFactory.java | 49 ---
.../execute/AsyncTaskCallbackFunctionImpl.java | 80 -----
.../runner/execute/AsyncTaskExecutionContext.java | 73 ----
.../execute/IMasterTaskExecutorThreadPool.java | 25 --
.../execute/MasterAsyncTaskExecutorThreadPool.java | 58 ----
.../execute/MasterSyncTaskExecutorThreadPool.java | 53 ---
.../execute/MasterTaskExecutionContextHolder.java | 47 ---
.../master/runner/execute/MasterTaskExecutor.java | 203 -----------
.../execute/MasterTaskExecutorFactoryBuilder.java | 54 ---
.../runner/execute/MasterTaskExecutorHolder.java | 50 ---
.../MasterTaskExecutorThreadPoolManager.java | 71 ----
.../runner/execute/SyncMasterTaskExecutor.java | 62 ----
.../execute/SyncMasterTaskExecutorFactory.java | 45 ---
...ogicTaskInstanceExecuteDispatchEventSender.java | 53 ---
...LogicTaskInstanceExecuteRunningEventSender.java | 57 ----
.../LogicTaskInstanceExecutionEventSender.java | 40 ---
...gicTaskInstanceExecutionEventSenderManager.java | 71 ----
.../master/runner/task/BaseAsyncLogicTask.java | 47 ---
.../master/runner/task/BaseSyncLogicTask.java | 71 ----
.../server/master/runner/task/IAsyncLogicTask.java | 26 --
.../src/main/resources/application.yaml | 7 +-
.../master/AbstractMasterIntegrationTestCase.java | 16 +
.../server/master/integration/MasterContainer.java | 87 +++++
.../master/integration/WorkflowOperator.java | 6 +
.../integration/WorkflowTestCaseContext.java | 9 +
.../cases/WorkflowBackfillTestCase.java | 33 +-
.../cases/WorkflowInstanceFailoverTestCase.java | 306 ++++++-----------
.../cases/WorkflowInstancePauseTestCase.java | 137 ++++----
...WorkflowInstanceRecoverFailureTaskTestCase.java | 43 +--
.../WorkflowInstanceRecoverPauseTestCase.java | 43 +--
.../cases/WorkflowInstanceRecoverStopTestCase.java | 45 +--
.../WorkflowInstanceRepeatRunningTestCase.java | 77 ++---
.../cases/WorkflowInstanceStopTestCase.java | 97 ++----
.../cases/WorkflowSchedulingTestCase.java | 31 +-
.../integration/cases/WorkflowStartTestCase.java | 91 ++---
.../GlobalTaskDispatchWaitingQueueLooperTest.java | 19 +-
.../dispatcher/MasterTaskDispatcherTest.java | 44 ---
.../runner/dispatcher/TaskDispatchFactoryTest.java | 64 ----
.../dispatcher/WorkerTaskDispatcherTest.java | 48 ---
.../DynamicAsyncTaskExecuteFunctionTest.java | 141 --------
.../task/dynamic/DynamicCommandUtilsTest.java | 135 --------
.../runner/task/dynamic/DynamicLogicTaskTest.java | 175 ----------
.../src/test/resources/application.yaml | 6 +-
.../workflow_with_sub_workflow_task_success.yaml | 4 +-
...ith_three_parallel_three_fake_task_success.yaml | 6 +-
.../src/test/resources/logback.xml | 1 +
.../src/main/resources/application.yaml | 13 +-
dolphinscheduler-task-executor/pom.xml | 47 +++
.../task/executor/AbstractTaskExecutor.java | 138 ++++++++
.../task/executor/ITaskEngine.java | 65 ++++
.../task/executor/ITaskExecutor.java | 77 +++++
.../task/executor/ITaskExecutorFactory.java | 7 +-
.../task/executor/ITaskExecutorRepository.java | 20 +-
.../task/executor/ITaskExecutorStateTracker.java | 19 +-
.../dolphinscheduler/task/executor/TaskEngine.java | 95 ++++++
.../task/executor/TaskEngineBuilder.java | 23 +-
.../task/executor/TaskExecutorRepository.java | 62 ++++
.../task/executor/TaskExecutorState.java | 28 +-
.../task/executor/TaskExecutorStateMappings.java | 57 ++--
.../container/AbstractTaskExecutorContainer.java | 148 ++++++++
.../ExclusiveThreadTaskExecutorContainer.java | 29 +-
.../executor/container/ITaskExecutorContainer.java | 60 ++++
.../container/ITaskExecutorContainerProvider.java | 6 +-
.../SharedThreadTaskExecutorContainer.java | 28 +-
.../container/TaskExecutorAssignmentTable.java | 49 +++
.../container/TaskExecutorContainerConfig.java | 14 +-
.../eventbus/ITaskExecutorEventBusCoordinator.java | 25 +-
.../ITaskExecutorEventRemoteReporterClient.java | 14 +-
.../ITaskExecutorLifecycleEventReporter.java | 72 ++++
.../executor/eventbus/TaskExecutorEventBus.java | 27 +-
.../eventbus/TaskExecutorEventBusCoordinator.java | 203 +++++++++++
.../TaskExecutorLifecycleEventRemoteReporter.java | 295 ++++++++++++++++
.../events/AbstractTaskExecutorLifecycleEvent.java | 38 +--
.../IOperableTaskExecutorLifecycleEvent.java | 16 +-
.../IReportableTaskExecutorLifecycleEvent.java | 52 +++
.../events/ITaskExecutorLifecycleEvent.java | 18 +-
.../TaskExecutorDispatchedLifecycleEvent.java | 50 +--
.../events/TaskExecutorFailedLifecycleEvent.java | 57 ++--
.../events/TaskExecutorFinalizeLifecycleEvent.java | 30 +-
.../events/TaskExecutorKillLifecycleEvent.java | 43 ++-
.../events/TaskExecutorKilledLifecycleEvent.java | 51 +--
.../events/TaskExecutorLifecycleEventType.java | 25 +-
.../events/TaskExecutorPauseLifecycleEvent.java | 34 +-
.../events/TaskExecutorPausedLifecycleEvent.java | 51 +--
...xecutorRuntimeContextChangedLifecycleEvent.java | 63 ++--
.../events/TaskExecutorStartedLifecycleEvent.java | 67 ++++
.../events/TaskExecutorSuccessLifecycleEvent.java | 55 +--
.../exceptions/TaskExecutorNotFoundException.java | 15 +-
.../exceptions/TaskExecutorRuntimeException.java | 17 +-
.../TaskExecutorRuntimeExceptionEnum.java | 13 +-
.../ITaskExecutorLifecycleEventListener.java | 53 +++
.../TaskExecutorLifecycleEventListener.java | 126 +++++++
.../task/executor/log/TaskExecutorMDCUtils.java | 58 ++++
.../operations/TaskExecutorDispatchRequest.java | 8 +-
.../operations/TaskExecutorDispatchResponse.java | 16 +-
.../operations/TaskExecutorKillRequest.java | 11 +-
.../operations/TaskExecutorKillResponse.java | 13 +-
.../operations/TaskExecutorPauseRequest.java | 9 +-
.../operations/TaskExecutorPauseResponse.java | 12 +-
.../TaskExecutorReassignMasterRequest.java | 4 +-
.../TaskExecutorReassignMasterResponse.java | 15 +-
.../worker/AbstractTaskExecutorWorker.java | 91 +++++
.../task/executor/worker/ITaskExecutorWorker.java | 30 +-
.../task/executor/worker/TaskExecutorWorker.java | 158 +++++++++
.../task/executor/worker/TaskExecutorWorkers.java | 61 ++++
.../plugin/task/api/AbstractTask.java | 17 +-
.../plugin/task/api/TaskCallBack.java | 6 +-
.../plugin/task/api/TaskConstants.java | 1 +
.../plugin/task/api/TaskExecutionContext.java | 5 +-
.../shell/BaseLinuxShellInterceptorBuilder.java | 4 +-
dolphinscheduler-worker/pom.xml | 6 +
.../server/worker/WorkerServer.java | 55 +--
.../server/worker/config/PhysicalTaskConfig.java | 10 +-
.../server/worker/config/WorkerConfig.java | 11 +-
.../executor/PhysicalTaskEngineDelegator.java | 88 +++++
.../worker/executor/PhysicalTaskEngineFactory.java | 48 +++
.../worker/executor/PhysicalTaskExecutor.java | 142 ++++++++
.../executor/PhysicalTaskExecutorBuilder.java | 25 +-
.../PhysicalTaskExecutorContainerProvider.java | 45 +++
.../PhysicalTaskExecutorEventBusCoordinator.java | 28 +-
...sicalTaskExecutorEventRemoteReporterClient.java | 14 +-
.../executor/PhysicalTaskExecutorFactory.java | 66 ++++
...PhysicalTaskExecutorLifecycleEventListener.java | 36 ++
...PhysicalTaskExecutorLifecycleEventReporter.java | 24 +-
.../executor/PhysicalTaskExecutorRepository.java | 14 +-
.../worker/executor/PhysicalTaskPluginFactory.java | 33 +-
.../server/worker/message/MessageRetryRunner.java | 196 -----------
.../message/TaskExecutionDispatchEventSender.java | 55 ---
.../message/TaskExecutionFailedEventSender.java | 56 ----
.../message/TaskExecutionKilledEventSender.java | 56 ----
.../message/TaskExecutionPausedEventSender.java | 56 ----
.../TaskInstanceExecutionRunningEventSender.java | 61 ----
.../worker/registry/WorkerRegistryClient.java | 6 +-
.../rpc/PhysicalTaskExecutorOperatorImpl.java | 104 ++++++
.../rpc/StreamingTaskInstanceOperatorImpl.java | 17 +-
.../TaskInstanceExecutionEventAckListenerImpl.java | 139 --------
.../worker/rpc/TaskInstanceOperatorImpl.java | 63 ----
.../server/worker/rpc/WorkerLogServiceImpl.java | 23 --
.../server/worker/rpc/WorkerMessageSender.java | 73 ----
.../worker/runner/DefaultWorkerTaskExecutor.java | 63 ----
.../runner/DefaultWorkerTaskExecutorFactory.java | 61 ----
.../server/worker/runner/TaskCallbackImpl.java | 56 ----
.../server/worker/runner/WorkerTaskExecutor.java | 373 ---------------------
.../runner/WorkerTaskExecutorFactoryBuilder.java | 64 ----
.../worker/runner/WorkerTaskExecutorHolder.java | 58 ----
.../runner/WorkerTaskExecutorThreadPool.java | 113 -------
.../operator/TakeOverTaskOperationFunction.java | 80 -----
.../TaskInstanceDispatchOperationFunction.java | 99 ------
.../TaskInstanceKillOperationFunction.java | 150 ---------
.../TaskInstanceOperationFunctionManager.java | 65 ----
.../TaskInstancePauseOperationFunction.java | 45 ---
.../server/worker/task/WorkerHeartBeatTask.java | 17 +-
.../src/main/resources/application.yaml | 5 +-
.../worker/registry/WorkerRegistryClientTest.java | 17 +-
.../runner/DefaultWorkerTaskExecutorTest.java | 87 -----
.../runner/WorkerTaskExecutorThreadPoolTest.java | 177 ----------
.../TaskInstanceOperationFunctionTest.java | 275 ---------------
pom.xml | 18 +
293 files changed, 5547 insertions(+), 8074 deletions(-)
diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md
index fb8613d388..5794152966 100644
--- a/docs/docs/en/architecture/configuration.md
+++ b/docs/docs/en/architecture/configuration.md
@@ -276,29 +276,21 @@ Location: `api-server/conf/application.yaml`
Location: `master-server/conf/application.yaml`
-| Parameters | Default value | Description |
-|-----------------------------------------------------------------------------|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| master.listen-port | 5678 | master listen port |
-| master.pre-exec-threads | 10 | master prepare execute thread number to limit handle commands in parallel |
-| master.exec-threads | 100 | master execute thread number to limit process instances in parallel |
-| master.dispatch-task-number | 3 | master dispatch task number per batch |
-| master.worker-load-balancer-configuration-properties.type | DYNAMIC_WEIGHTED_ROUND_ROBIN | Master will use the worker's cpu/memory/threadPool usage to calculate the worker load, the lower load will have more change to be dispatched task |
-| master.max-heartbeat-interval | 10s | master max heartbeat interval |
-| master.task-commit-retry-times | 5 | master commit task retry times |
-| master.task-commit-interval | 1000 | master commit task interval, the unit is millisecond |
-| master.state-wheel-interval | 5 | time to check status |
-| master.server-load-protection.enabled | true | If set true, will open master overload protection |
-| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. |
-| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | Master max JVM cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. |
-| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max system memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. |
-| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. |
-| master.failover-interval | 10 | failover interval, the unit is minute |
-| master.kill-application-when-task-failover | true | whether to kill yarn/k8s application when failover taskInstance |
-| master.registry-disconnect-strategy.max-waiting-time | 100s | Used when the master disconnect from registry, and the disconnect strategy is waiting, this config means the master will waiting to reconnect to registry in given times, and after the waiting times, if the master still cannot connect to registry, will stop itself, if the value is 0s, the Master will wait infinitely |
-| master.worker-group-refresh-interval | 10s | The interval to refresh worker group from db to memory |
-| master.command-fetch-strategy.type | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED` |
-| master.command-fetch-strategy.config.id-step | 1 | The id auto incremental step of t_ds_command in db |
-| master.command-fetch-strategy.config.fetch-size | 10 | The number of commands fetched by master |
+| Parameters | Default value | Description |
+|-----------------------------------------------------------------------------|------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|
+| master.listen-port | 5678 | master listen port |
+| master.logic-task-config.task-executor-thread-count | 2 * CPU +1 | The thread size used to execute logic task |
+| master.worker-load-balancer-configuration-properties.type | DYNAMIC_WEIGHTED_ROUND_ROBIN | Master will use the worker's cpu/memory/threadPool usage to calculate the worker load, the lower load will have more change to be dispatched task |
+| master.max-heartbeat-interval | 10s | master max heartbeat interval |
+| master.server-load-protection.enabled | true | If set true, will open master overload protection |
+| master.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. |
+| master.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | Master max JVM cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. |
+| master.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | Master max system memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. |
+| master.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. |
+| master.worker-group-refresh-interval | 10s | The interval to refresh worker group from db to memory |
+| master.command-fetch-strategy.type | ID_SLOT_BASED | The command fetch strategy, only support `ID_SLOT_BASED` |
+| master.command-fetch-strategy.config.id-step | 1 | The id auto incremental step of t_ds_command in db |
+| master.command-fetch-strategy.config.fetch-size | 10 | The number of commands fetched by master |
### Worker Server related configuration
@@ -307,7 +299,6 @@ Location: `worker-server/conf/application.yaml`
| Parameters | Default value | Description |
|-----------------------------------------------------------------------------|---------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| worker.listen-port | 1234 | worker-service listen port |
-| worker.exec-threads | 100 | worker-service execute thread number, used to limit the number of task instances in parallel |
| worker.max-heartbeat-interval | 10s | worker-service max heartbeat interval |
| worker.host-weight | 100 | worker host weight to dispatch tasks |
| worker.server-load-protection.enabled | true | If set true will open worker overload protection |
@@ -317,7 +308,7 @@ Location: `worker-server/conf/application.yaml`
| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | Worker max disk usage , when the worker's disk usage is smaller then this value, master server can execute workflow. |
| worker.registry-disconnect-strategy.strategy | stop | Used when the worker disconnect from registry, default value: stop. Optional values include stop, waiting |
| worker.registry-disconnect-strategy.max-waiting-time | 100s | Used when the worker disconnect from registry, and the disconnect strategy is waiting, this config means the worker will waiting to reconnect to registry in given times, and after the waiting times, if the worker still cannot connect to registry, will stop itself, if the value is 0s, will wait infinitely |
-| worker.task-execute-threads-full-policy | REJECT | If REJECT, when the task waiting in the worker reaches exec-threads, it will reject the received task and the Master will redispatch it; If CONTINUE, it will put the task into the worker's execution queue and wait for a free thread to start execution |
+| worker.physical-task-config.task-executor-thread-size | 100 | The thread size used to execute physical task |
| worker.tenant-config.auto-create-tenant-enabled | true | tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. |
| worker.tenant-config.default-tenant-enabled | false | If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. |
diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md
index 91abd04811..12dd7fa356 100644
--- a/docs/docs/en/guide/upgrade/incompatible.md
+++ b/docs/docs/en/guide/upgrade/incompatible.md
@@ -34,4 +34,5 @@ This document records the incompatible updates between each version. You need to
* Deprecated upgrade code of 1.x and 2.x ([#16543])(https://github.com/apache/dolphinscheduler/pull/16543)
* Remove the `Data Quality` module ([#16794])(https://github.com/apache/dolphinscheduler/pull/16794)
* Remove the `registry-disconnect-strategy` in `application.yaml` ([#16821])(https://github.com/apache/dolphinscheduler/pull/16821)
+* Remove `exec-threads` in worker's `application.yaml`, please use `physical-task-config`;Remove `master-async-task-executor-thread-pool-size` in master's `application.yaml`, please use `logic-task-config` ([#16790])(https://github.com/apache/dolphinscheduler/pull/16790)
diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md
index c76e869f72..d167bba45e 100644
--- a/docs/docs/zh/architecture/configuration.md
+++ b/docs/docs/zh/architecture/configuration.md
@@ -303,24 +303,22 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
位置：`worker-server/conf/application.yaml`
-| 参数 | 默认值 | 描述 |
-|-----------------------------------------------------------------------------|-----------|-------------------------------------------------------------------------------------------------------------------------------------------|
-| worker.listen-port | 1234 | worker监听端口 |
-| worker.exec-threads | 100 | worker工作线程数量,用于限制并行的任务实例数量 |
-| worker.max-heartbeat-interval | 10s | worker最大心跳间隔 |
-| worker.host-weight | 100 | 派发任务时，worker主机的权重 |
-| worker.tenant-auto-create | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
-| worker.server-load-protection.enabled | true | 是否开启系统保护策略 |
-| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | worker最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统CPU |
-| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | worker最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的JVM CPU |
-| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | worker最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统内存 |
-| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | worker最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 |
-| worker.alert-listen-host | localhost | alert监听host |
-| worker.alert-listen-port | 50052 | alert监听端口 |
-| worker.registry-disconnect-strategy.max-waiting-time | 100s | 当Worker与注册中心失联之后重连时间, 之后当strategy为waiting时，该值生效。 该值表示当Worker与注册中心失联时会在给定时间之内进行重连, 在给定时间之内重连失败将会停止自己，在重连时，Worker会丢弃kill正在执行的任务。值为0表示会无限期等待 |
-| worker.task-execute-threads-full-policy | REJECT | 如果是 REJECT, 当Worker中等待队列中的任务数达到exec-threads时, Worker将会拒绝接下来新接收的任务，Master将会重新分发该任务; 如果是 CONTINUE, Worker将会接收任务，放入等待队列中等待空闲线程去执行该任务 |
-| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
-| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 |
+| 参数 | 默认值 | 描述 |
+|-----------------------------------------------------------------------------|-----------|-----------------------------------------------------------------------------------------|
+| worker.listen-port | 1234 | worker监听端口 |
+| worker.max-heartbeat-interval | 10s | worker最大心跳间隔 |
+| worker.host-weight | 100 | 派发任务时，worker主机的权重 |
+| worker.tenant-auto-create | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
+| worker.server-load-protection.enabled | true | 是否开启系统保护策略 |
+| worker.server-load-protection.max-system-cpu-usage-percentage-thresholds | 0.7 | worker最大系统cpu使用值,只有当前系统cpu使用值低于最大系统cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统CPU |
+| worker.server-load-protection.max-jvm-cpu-usage-percentage-thresholds | 0.7 | worker最大JVM cpu使用值,只有当前JVM cpu使用值低于最大JVM cpu使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的JVM CPU |
+| worker.server-load-protection.max-system-memory-usage-percentage-thresholds | 0.7 | worker最大系统 内存使用值,只有当前系统内存使用值低于最大系统内存使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统内存 |
+| worker.server-load-protection.max-disk-usage-percentage-thresholds | 0.7 | worker最大系统磁盘使用值,只有当前系统磁盘使用值低于最大系统磁盘使用值,worker服务才能接收任务. 默认值为0.7: 会使用70%的操作系统磁盘空间 |
+| worker.alert-listen-host | localhost | alert监听host |
+| worker.alert-listen-port | 50052 | alert监听端口 |
+| worker.physical-task-config.task-executor-thread-size | 100 | Worker中任务最大并发度 |
+| worker.tenant-config.auto-create-tenant-enabled | true | 租户对应于系统的用户,由worker提交作业.如果系统没有该用户,则在参数worker.tenant.auto.create为true后自动创建。 |
+| worker.tenant-config.default-tenant-enabled | false | 如果设置为true, 将会使用worker服务启动用户作为 `default` 租户。 |
## Alert Server相关配置
diff --git a/docs/docs/zh/guide/upgrade/incompatible.md b/docs/docs/zh/guide/upgrade/incompatible.md
index 4ba7d4bf95..412a66e4b3 100644
--- a/docs/docs/zh/guide/upgrade/incompatible.md
+++ b/docs/docs/zh/guide/upgrade/incompatible.md
@@ -32,4 +32,5 @@
* 废弃从 1.x 至 2.x 的升级代码 ([#16543])(https://github.com/apache/dolphinscheduler/pull/16543)
* 移除 `数据质量` 模块 ([#16794])(https://github.com/apache/dolphinscheduler/pull/16794)
* 在`application.yaml`中移除`registry-disconnect-strategy`配置 ([#16821])(https://github.com/apache/dolphinscheduler/pull/16821)
+* 在worker的`application.yaml`中移除`exec-threads`，使用`physical-task-config`替代;在master的`application.yaml`中移除`master-async-task-executor-thread-pool-size`使用`logic-task-config`替代 ([#16790])(https://github.com/apache/dolphinscheduler/pull/16790)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
index 957aca912f..372ee35b6d 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
@@ -46,14 +46,14 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao;
import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils;
import org.apache.dolphinscheduler.extract.base.client.Clients;
import org.apache.dolphinscheduler.extract.common.ILogService;
+import org.apache.dolphinscheduler.extract.worker.IPhysicalTaskExecutorOperator;
import org.apache.dolphinscheduler.extract.worker.IStreamingTaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointResponse;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillResponse;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
@@ -302,11 +302,11 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst
}
// todo: we only support streaming task for now
- final TaskInstanceKillResponse taskInstanceKillResponse = Clients
- .withService(ITaskInstanceOperator.class)
+ final TaskExecutorKillResponse taskExecutorKillResponse = Clients
+ .withService(IPhysicalTaskExecutorOperator.class)
.withHost(taskInstance.getHost())
- .killTask(new TaskInstanceKillRequest(taskInstanceId));
- log.info("TaskInstance kill response: {}", taskInstanceKillResponse);
+ .killTask(TaskExecutorKillRequest.of(taskInstanceId));
+ log.info("TaskInstance kill response: {}", taskExecutorKillResponse);
putMsg(result, Status.SUCCESS);
return result;
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java
index c6ced818c6..306cf91de2 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java
@@ -43,8 +43,6 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig;
import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery;
import org.apache.dolphinscheduler.extract.common.ILogService;
-import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdRequest;
-import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdResponse;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest;
@@ -131,11 +129,6 @@ public class LoggerServiceTest {
return new TaskInstanceLogPageQueryResponse();
}
- @Override
- public GetAppIdResponse getAppId(GetAppIdRequest getAppIdRequest) {
- return new GetAppIdResponse();
- }
-
@Override
public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) {
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
index 932da89d6b..1c3cfc86b7 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/thread/ThreadUtils.java
@@ -31,6 +31,12 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
@Slf4j
public class ThreadUtils {
+ /**
+ * Create a daemon fixed thread pool, the thread name will be formatted with the given name.
+ *
+ * @param threadNameFormat the thread name format, e.g. "DemonThread-%d"
+ * @param threadsNum the number of threads in the pool
+ */
public static ThreadPoolExecutor newDaemonFixedThreadExecutor(String threadNameFormat, int threadsNum) {
return (ThreadPoolExecutor) Executors.newFixedThreadPool(threadsNum, newDaemonThreadFactory(threadNameFormat));
}
@@ -43,9 +49,10 @@ public class ThreadUtils {
* Create a daemon scheduler thread pool, the thread name will be formatted with the given name.
*
* @param threadNameFormat the thread name format, e.g. "DemonThread-%d"
- * @param threadsNum the number of threads in the pool
+ * @param threadsNum the number of threads in the pool
*/
- public static ScheduledExecutorService newDaemonScheduledExecutorService(String threadNameFormat, int threadsNum) {
+ public static ScheduledExecutorService newDaemonScheduledExecutorService(final String threadNameFormat,
+ final int threadsNum) {
return Executors.newScheduledThreadPool(threadsNum, newDaemonThreadFactory(threadNameFormat));
}
diff --git a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java
index bc57aef9e9..c4c61b9055 100644
--- a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java
+++ b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEvent.java
@@ -20,37 +20,47 @@ package org.apache.dolphinscheduler.eventbus;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
+import lombok.Builder;
+import lombok.ToString;
+import lombok.experimental.SuperBuilder;
+
/**
* The abstract class of delay event, the event will be triggered after the delay time.
* <p> You can extend this class to implement your own delay event.
*/
+@ToString
+@SuperBuilder
public abstract class AbstractDelayEvent implements IEvent, Delayed {
+ private static final long DEFAULT_DELAY_TIME = 0;
+
protected long delayTime;
- protected long triggerTimeInMillis;
+ @Builder.Default
+ protected long createTimeInNano = System.nanoTime();
public AbstractDelayEvent() {
- this(0);
+ this(DEFAULT_DELAY_TIME);
+ }
+
+ public AbstractDelayEvent(final long delayTime) {
+ this(delayTime, System.nanoTime());
}
- public AbstractDelayEvent(long delayTime) {
- if (delayTime == 0) {
- this.triggerTimeInMillis = System.currentTimeMillis();
- } else {
- this.triggerTimeInMillis = System.currentTimeMillis() + delayTime;
- }
+ public AbstractDelayEvent(final long delayTime, final long createTimeInNano) {
+ this.delayTime = delayTime;
+ this.createTimeInNano = createTimeInNano;
}
@Override
public long getDelay(TimeUnit unit) {
- long delay = triggerTimeInMillis - System.currentTimeMillis();
- return unit.convert(delay, TimeUnit.MILLISECONDS);
+ long delay = createTimeInNano + delayTime * 1_000_000 - System.nanoTime();
+ return unit.convert(delay, TimeUnit.NANOSECONDS);
}
@Override
public int compareTo(Delayed other) {
- return Long.compare(this.triggerTimeInMillis, ((AbstractDelayEvent) other).triggerTimeInMillis);
+ return Long.compare(this.createTimeInNano, ((AbstractDelayEvent) other).createTimeInNano);
}
}
diff --git a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEventBus.java b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEventBus.java
index 4d3c259e45..81809c546c 100644
--- a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEventBus.java
+++ b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/AbstractDelayEventBus.java
@@ -37,6 +37,16 @@ public abstract class AbstractDelayEventBus<T extends AbstractDelayEvent> implem
return Optional.ofNullable(delayEventQueue.poll());
}
+ @Override
+ public Optional<T> peek() {
+ return Optional.ofNullable(delayEventQueue.peek());
+ }
+
+ @Override
+ public Optional<T> remove() {
+ return Optional.ofNullable(delayEventQueue.remove());
+ }
+
@Override
public boolean isEmpty() {
return delayEventQueue.isEmpty();
diff --git a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/IEventBus.java b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/IEventBus.java
index 8a8c458e6a..b2f6577a41 100644
--- a/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/IEventBus.java
+++ b/dolphinscheduler-eventbus/src/main/java/org/apache/dolphinscheduler/eventbus/IEventBus.java
@@ -54,6 +54,16 @@ public interface IEventBus<T extends IEvent> {
*/
Optional<T> poll() throws InterruptedException;
+ /**
+ * peek the head event from the bus. This method will not block if the event bus is empty will return empty optional.
+ */
+ Optional<T> peek();
+
+ /**
+ * Remove the head event from the bus. This method will not block if the event bus is empty will return empty optional.
+ */
+ Optional<T> remove();
+
/**
* Whether the bus is empty.
*/
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml
index 0554ea3c3c..8a03faf409 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml
@@ -45,27 +45,28 @@
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-common</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-meter</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
+
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
+
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java
index 56e19fe8b2..3450192f62 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java
@@ -25,12 +25,12 @@ import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYY_MM
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.TimeZone;
+import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -38,9 +38,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
-/**
- * json serialize or deserialize
- */
@Slf4j
public class JsonSerializer {
@@ -60,13 +57,6 @@ public class JsonSerializer {
}
- /**
- * serialize to byte
- *
- * @param obj object
- * @param <T> object type
- * @return byte array
- */
public static <T> byte[] serialize(T obj) {
if (obj == null) {
return null;
@@ -79,44 +69,14 @@ public class JsonSerializer {
}
}
- /**
- * serialize to string
- *
- * @param obj object
- * @param <T> object type
- * @return string
- */
- public static <T> String serializeToString(T obj) {
- String json = "";
- try {
- json = objectMapper.writeValueAsString(obj);
- } catch (JsonProcessingException e) {
- log.error("serializeToString exception!", e);
- }
-
- return json;
- }
-
- /**
- * deserialize
- *
- * @param src byte array
- * @param clazz class
- * @param <T> deserialize type
- * @return deserialize type
- */
+ @SneakyThrows
public static <T> T deserialize(byte[] src, Class<T> clazz) {
if (src == null) {
return null;
}
String json = new String(src, StandardCharsets.UTF_8);
- try {
- return objectMapper.readValue(json, clazz);
- } catch (IOException e) {
- log.error("deserialize exception!", e);
- return null;
- }
+ return objectMapper.readValue(json, clazz);
}
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
index 9ebf802b1e..246e11735c 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java
@@ -70,12 +70,12 @@ class NettyRemotingServer {
this.serverConfig = serverConfig;
this.serverName = serverConfig.getServerName();
this.methodInvokerExecutor = ThreadUtils.newDaemonFixedThreadExecutor(
- serverName + "MethodInvoker-%d", Runtime.getRuntime().availableProcessors() * 2 + 1);
+ serverName + "-methodInvoker-%d", Runtime.getRuntime().availableProcessors() * 2 + 1);
this.channelHandler = new JdkDynamicServerHandler(methodInvokerExecutor);
ThreadFactory bossThreadFactory =
- ThreadUtils.newDaemonThreadFactory(serverName + "BossThread-%d");
+ ThreadUtils.newDaemonThreadFactory(serverName + "-boss-%d");
ThreadFactory workerThreadFactory =
- ThreadUtils.newDaemonThreadFactory(serverName + "WorkerThread-%d");
+ ThreadUtils.newDaemonThreadFactory(serverName + "-worker-%d");
if (Epoll.isAvailable()) {
this.bossGroup = new EpollEventLoopGroup(1, bossThreadFactory);
this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), workerThreadFactory);
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-common/pom.xml
index d7d339ff9f..f763ec22c0 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-common/pom.xml
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/pom.xml
@@ -27,14 +27,12 @@
<version>dev-SNAPSHOT</version>
</parent>
- <groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-common</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-base</artifactId>
- <version>${project.version}</version>
</dependency>
</dependencies>
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java
index c602f15bb2..4484e51942 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-common/src/main/java/org/apache/dolphinscheduler/extract/common/ILogService.java
@@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.extract.common;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
-import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdRequest;
-import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdResponse;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest;
@@ -35,9 +33,6 @@ public interface ILogService {
@RpcMethod
TaskInstanceLogPageQueryResponse pageQueryTaskInstanceLog(TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest);
- @RpcMethod
- GetAppIdResponse getAppId(GetAppIdRequest getAppIdRequest);
-
@RpcMethod
void removeTaskInstanceLog(String taskInstanceLogAbsolutePath);
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml
index 955bcd0a12..e210fd9056 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml
@@ -30,21 +30,27 @@
<artifactId>dolphinscheduler-extract-master</artifactId>
<dependencies>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-common</artifactId>
- <version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-base</artifactId>
- <version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-task-api</artifactId>
- <version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-task-executor</artifactId>
+ </dependency>
+
</dependencies>
</project>
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskExecutorOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskExecutorOperator.java
new file mode 100644
index 0000000000..1344a29ce1
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskExecutorOperator.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.master;
+
+import org.apache.dolphinscheduler.extract.base.RpcMethod;
+import org.apache.dolphinscheduler.extract.base.RpcService;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseResponse;
+
+@RpcService
+public interface ILogicTaskExecutorOperator {
+
+ @RpcMethod
+ TaskExecutorDispatchResponse dispatchTask(final TaskExecutorDispatchRequest taskExecutorDispatchRequest);
+
+ @RpcMethod
+ TaskExecutorKillResponse killTask(final TaskExecutorKillRequest taskExecutorKillRequest);
+
+ @RpcMethod
+ TaskExecutorPauseResponse pauseTask(final TaskExecutorPauseRequest taskExecutorPauseRequest);
+
+ @RpcMethod
+ void ackTaskExecutorLifecycleEvent(final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck);
+
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java
deleted file mode 100644
index f85200b7a2..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.master;
-
-import org.apache.dolphinscheduler.extract.base.RpcMethod;
-import org.apache.dolphinscheduler.extract.base.RpcService;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse;
-
-@RpcService
-public interface ILogicTaskInstanceOperator {
-
- @RpcMethod
- LogicTaskDispatchResponse dispatchLogicTask(LogicTaskDispatchRequest taskDispatchRequest);
-
- @RpcMethod
- LogicTaskKillResponse killLogicTask(LogicTaskKillRequest taskKillRequest);
-
- @RpcMethod
- LogicTaskPauseResponse pauseLogicTask(LogicTaskPauseRequest taskPauseRequest);
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskExecutionEventListener.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskExecutionEventListener.java
deleted file mode 100644
index 7ebd1310bd..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskExecutionEventListener.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.master;
-
-import org.apache.dolphinscheduler.extract.base.RpcMethod;
-import org.apache.dolphinscheduler.extract.base.RpcService;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionDispatchEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionFailedEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionKilledEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionPausedEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionRunningEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionSuccessEvent;
-
-@RpcService
-public interface ITaskExecutionEventListener {
-
- @RpcMethod
- void onTaskInstanceDispatched(final TaskExecutionDispatchEvent taskExecutionDispatchEvent);
-
- @RpcMethod
- void onTaskInstanceExecutionRunning(final TaskExecutionRunningEvent taskInstanceExecutionRunningEvent);
-
- @RpcMethod
- void onTaskInstanceExecutionSuccess(final TaskExecutionSuccessEvent taskInstanceExecutionSuccessEvent);
-
- @RpcMethod
- void onTaskInstanceExecutionFailed(final TaskExecutionFailedEvent taskInstanceExecutionFailedEvent);
-
- @RpcMethod
- void onTaskInstanceExecutionKilled(final TaskExecutionKilledEvent taskInstanceExecutionKilledEvent);
-
- @RpcMethod
- void onTaskInstanceExecutionPaused(final TaskExecutionPausedEvent taskInstanceExecutionPausedEvent);
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskExecutorEventListener.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskExecutorEventListener.java
new file mode 100644
index 0000000000..86b8f66568
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskExecutorEventListener.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.master;
+
+import org.apache.dolphinscheduler.extract.base.RpcMethod;
+import org.apache.dolphinscheduler.extract.base.RpcService;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorDispatchedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorFailedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorKilledLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorPausedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorRuntimeContextChangedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorStartedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorSuccessLifecycleEvent;
+
+@RpcService
+public interface ITaskExecutorEventListener {
+
+ @RpcMethod
+ void onTaskExecutorDispatched(final TaskExecutorDispatchedLifecycleEvent taskExecutorDispatchedLifecycleEvent);
+
+ @RpcMethod
+ void onTaskExecutorRunning(final TaskExecutorStartedLifecycleEvent taskExecutorStartedLifecycleEvent);
+
+ @RpcMethod
+ void onTaskExecutorRuntimeContextChanged(final TaskExecutorRuntimeContextChangedLifecycleEvent taskExecutorRuntimeContextChangedLifecycleEventr);
+
+ @RpcMethod
+ void onTaskExecutorSuccess(final TaskExecutorSuccessLifecycleEvent taskExecutorSuccessLifecycleEvent);
+
+ @RpcMethod
+ void onTaskExecutorFailed(final TaskExecutorFailedLifecycleEvent taskExecutorFailedLifecycleEvent);
+
+ @RpcMethod
+ void onTaskExecutorKilled(final TaskExecutorKilledLifecycleEvent taskExecutorKilledLifecycleEvent);
+
+ @RpcMethod
+ void onTaskExecutorPaused(final TaskExecutorPausedLifecycleEvent taskExecutorPausedLifecycleEvent);
+
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/TaskExecutorEventRemoteReporterClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/TaskExecutorEventRemoteReporterClient.java
new file mode 100644
index 0000000000..955e44c662
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/TaskExecutorEventRemoteReporterClient.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.master;
+
+import org.apache.dolphinscheduler.extract.base.client.Clients;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorEventRemoteReporterClient;
+import org.apache.dolphinscheduler.task.executor.events.IReportableTaskExecutorLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorDispatchedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorFailedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorKilledLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorPausedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorRuntimeContextChangedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorStartedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorSuccessLifecycleEvent;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class TaskExecutorEventRemoteReporterClient implements ITaskExecutorEventRemoteReporterClient {
+
+ public void reportTaskExecutionEventToMaster(final IReportableTaskExecutorLifecycleEvent taskExecutorLifecycleEvent) {
+ try {
+ taskExecutorLifecycleEvent.setLatestReportTime(System.currentTimeMillis());
+ switch (taskExecutorLifecycleEvent.getType()) {
+ case DISPATCHED:
+ reportTaskDispatchedEventToMaster(
+ (TaskExecutorDispatchedLifecycleEvent) taskExecutorLifecycleEvent);
+ break;
+ case RUNNING:
+ reportTaskRunningEventToMaster((TaskExecutorStartedLifecycleEvent) taskExecutorLifecycleEvent);
+ break;
+ case RUNTIME_CONTEXT_CHANGE:
+ reportTaskRuntimeContextChangeEventToMaster(
+ (TaskExecutorRuntimeContextChangedLifecycleEvent) taskExecutorLifecycleEvent);
+ break;
+ case PAUSED:
+ reportTaskPausedEventToMaster((TaskExecutorPausedLifecycleEvent) taskExecutorLifecycleEvent);
+ break;
+ case KILLED:
+ reportTaskKilledEventToMaster((TaskExecutorKilledLifecycleEvent) taskExecutorLifecycleEvent);
+ break;
+ case FAILED:
+ reportTaskFailedEventToMaster((TaskExecutorFailedLifecycleEvent) taskExecutorLifecycleEvent);
+ break;
+ case SUCCESS:
+ reportTaskSuccessEventToMaster((TaskExecutorSuccessLifecycleEvent) taskExecutorLifecycleEvent);
+ break;
+ default:
+ log.warn("Unsupported TaskExecutionEvent: {}", taskExecutorLifecycleEvent);
+ }
+ log.info("Report: {} to master success", taskExecutorLifecycleEvent);
+ } catch (Throwable throwable) {
+ log.error("Report ITaskExecutorLifecycleEvent: {} to master failed", taskExecutorLifecycleEvent, throwable);
+ }
+ }
+
+ private static void reportTaskDispatchedEventToMaster(final TaskExecutorDispatchedLifecycleEvent taskExecutionDispatchedEvent) {
+ Clients
+ .withService(ITaskExecutorEventListener.class)
+ .withHost(taskExecutionDispatchedEvent.getWorkflowInstanceHost())
+ .onTaskExecutorDispatched(taskExecutionDispatchedEvent);
+ }
+
+ private static void reportTaskRunningEventToMaster(final TaskExecutorStartedLifecycleEvent taskExecutionRunningEvent) {
+ Clients
+ .withService(ITaskExecutorEventListener.class)
+ .withHost(taskExecutionRunningEvent.getWorkflowInstanceHost())
+ .onTaskExecutorRunning(taskExecutionRunningEvent);
+ }
+
+ private static void reportTaskRuntimeContextChangeEventToMaster(final TaskExecutorRuntimeContextChangedLifecycleEvent taskExecutorLifecycleEvent) {
+ Clients
+ .withService(ITaskExecutorEventListener.class)
+ .withHost(taskExecutorLifecycleEvent.getWorkflowInstanceHost())
+ .onTaskExecutorRuntimeContextChanged(taskExecutorLifecycleEvent);
+ }
+
+ private static void reportTaskPausedEventToMaster(final TaskExecutorPausedLifecycleEvent taskExecutionPausedEvent) {
+ Clients
+ .withService(ITaskExecutorEventListener.class)
+ .withHost(taskExecutionPausedEvent.getWorkflowInstanceHost())
+ .onTaskExecutorPaused(taskExecutionPausedEvent);
+ }
+
+ private static void reportTaskKilledEventToMaster(final TaskExecutorKilledLifecycleEvent taskExecutionKilledEvent) {
+ Clients
+ .withService(ITaskExecutorEventListener.class)
+ .withHost(taskExecutionKilledEvent.getWorkflowInstanceHost())
+ .onTaskExecutorKilled(taskExecutionKilledEvent);
+ }
+
+ private static void reportTaskFailedEventToMaster(final TaskExecutorFailedLifecycleEvent taskExecutionFailedEvent) {
+ Clients
+ .withService(ITaskExecutorEventListener.class)
+ .withHost(taskExecutionFailedEvent.getWorkflowInstanceHost())
+ .onTaskExecutorFailed(taskExecutionFailedEvent);
+ }
+
+ private static void reportTaskSuccessEventToMaster(final TaskExecutorSuccessLifecycleEvent taskExecutionSuccessEvent) {
+ Clients
+ .withService(ITaskExecutorEventListener.class)
+ .withHost(taskExecutionSuccessEvent.getWorkflowInstanceHost())
+ .onTaskExecutorSuccess(taskExecutionSuccessEvent);
+ }
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskExecutionEvent.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskExecutionEvent.java
index a1df0829a5..8d9f4f7a90 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskExecutionEvent.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskExecutionEvent.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.extract.master.transportor;
// todo: add event source to distinguish the event is from executor or user operation
+// 将这个类移到common中?
public interface ITaskExecutionEvent {
int getWorkflowInstanceId();
@@ -28,9 +29,9 @@ public interface ITaskExecutionEvent {
void setEventCreateTime(long eventCreateTime);
- long getEventSendTime();
+ Long getEventSendTime();
- void setEventSendTime(long eventSendTime);
+ void setEventSendTime(Long eventSendTime);
void setWorkflowInstanceHost(String host);
@@ -43,7 +44,7 @@ public interface ITaskExecutionEvent {
TaskInstanceExecutionEventType getEventType();
enum TaskInstanceExecutionEventType {
- DISPATCH,
+ DISPATCHED,
RUNNING,
PAUSED,
KILLED,
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerRequest.java
deleted file mode 100644
index 7f2b16a530..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/StreamingTaskTriggerRequest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.master.transportor;
-
-import java.util.Map;
-
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-public class StreamingTaskTriggerRequest {
-
- private int executorId;
-
- private String executorName;
-
- private long projectCode;
-
- private long taskDefinitionCode;
-
- private int taskDefinitionVersion;
-
- private int warningGroupId;
-
- private String workerGroup;
-
- private Long environmentCode;
-
- private Map<String, String> startParams;
-
- private String tenantCode;
-
- private int dryRun;
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskExecutionSuccessEvent.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskExecutionSuccessEvent.java
deleted file mode 100644
index a7dd1c8909..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskExecutionSuccessEvent.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.master.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskExecutionSuccessEvent implements ITaskExecutionEvent {
-
- private int taskInstanceId;
-
- private int workflowInstanceId;
-
- private String taskInstanceHost;
-
- private String workflowInstanceHost;
-
- private long endTime;
-
- private int processId;
-
- private String appIds;
-
- private String varPool;
-
- private long eventCreateTime;
-
- private long eventSendTime;
-
- @Override
- public TaskInstanceExecutionEventType getEventType() {
- return TaskInstanceExecutionEventType.SUCCESS;
- }
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml b/dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml
index 0a42b73a1d..57d3857292 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/pom.xml
@@ -30,11 +30,12 @@
<artifactId>dolphinscheduler-extract-worker</artifactId>
<dependencies>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-common</artifactId>
- <version>${project.version}</version>
</dependency>
+
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-base</artifactId>
@@ -45,6 +46,11 @@
<artifactId>dolphinscheduler-task-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-task-executor</artifactId>
+ </dependency>
+
</dependencies>
</project>
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IPhysicalTaskExecutorOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IPhysicalTaskExecutorOperator.java
new file mode 100644
index 0000000000..b0cf4538ea
--- /dev/null
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/IPhysicalTaskExecutorOperator.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.extract.worker;
+
+import org.apache.dolphinscheduler.extract.base.RpcMethod;
+import org.apache.dolphinscheduler.extract.base.RpcService;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorReassignMasterRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorReassignMasterResponse;
+
+@RpcService
+public interface IPhysicalTaskExecutorOperator {
+
+ @RpcMethod
+ TaskExecutorDispatchResponse dispatchTask(final TaskExecutorDispatchRequest taskExecutorDispatchRequest);
+
+ @RpcMethod
+ TaskExecutorKillResponse killTask(final TaskExecutorKillRequest taskExecutorKillRequest);
+
+ @RpcMethod
+ TaskExecutorPauseResponse pauseTask(final TaskExecutorPauseRequest taskExecutorPauseRequest);
+
+ @RpcMethod
+ TaskExecutorReassignMasterResponse reassignWorkflowInstanceHost(final TaskExecutorReassignMasterRequest taskExecutorReassignMasterRequest);
+
+ @RpcMethod
+ void ackPhysicalTaskExecutorLifecycleEvent(final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck);
+
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceExecutionEventAckListener.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceExecutionEventAckListener.java
deleted file mode 100644
index fda644110f..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceExecutionEventAckListener.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker;
-
-import org.apache.dolphinscheduler.extract.base.RpcMethod;
-import org.apache.dolphinscheduler.extract.base.RpcService;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionFailedEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionKilledEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionPausedEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionSuccessEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionDispatchedEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck;
-
-@RpcService
-public interface ITaskInstanceExecutionEventAckListener {
-
- @RpcMethod
- void handleTaskInstanceDispatchedEventAck(final TaskInstanceExecutionDispatchedEventAck taskInstanceExecutionDispatchedEventAck);
-
- // todo: If we use sync, then we don't need ack here
- @RpcMethod
- void handleTaskInstanceExecutionRunningEventAck(TaskInstanceExecutionRunningEventAck taskInstanceExecutionRunningEventAck);
-
- @RpcMethod
- void handleTaskExecutionSuccessEventAck(TaskExecutionSuccessEventAck taskExecutionSuccessEventAck);
-
- @RpcMethod
- void handleTaskExecutionPausedEventAck(TaskExecutionPausedEventAck taskExecutionPausedEventAck);
-
- @RpcMethod
- void handleTaskExecutionFailedEventAck(TaskExecutionFailedEventAck taskExecutionFailedEventAck);
-
- @RpcMethod
- void handleTaskExecutionKilledEventAck(TaskExecutionKilledEventAck taskExecutionKilledEventAck);
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceOperator.java
deleted file mode 100644
index 43d6a39b01..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/ITaskInstanceOperator.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker;
-
-import org.apache.dolphinscheduler.extract.base.RpcMethod;
-import org.apache.dolphinscheduler.extract.base.RpcService;
-import org.apache.dolphinscheduler.extract.worker.transportor.TakeOverTaskRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TakeOverTaskResponse;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchResponse;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseResponse;
-
-@RpcService
-public interface ITaskInstanceOperator {
-
- @RpcMethod
- TaskInstanceDispatchResponse dispatchTask(final TaskInstanceDispatchRequest taskInstanceDispatchRequest);
-
- @RpcMethod
- TaskInstanceKillResponse killTask(final TaskInstanceKillRequest taskInstanceKillRequest);
-
- @RpcMethod
- TaskInstancePauseResponse pauseTask(final TaskInstancePauseRequest taskPauseRequest);
-
- @RpcMethod
- TakeOverTaskResponse takeOverTask(final TakeOverTaskRequest takeOverTaskRequest);
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionFailedEventAck.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionFailedEventAck.java
deleted file mode 100644
index 3c02bbe461..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionFailedEventAck.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskExecutionFailedEventAck {
-
- private int taskInstanceId;
- private boolean success;
-
- public static TaskExecutionFailedEventAck success(int taskInstanceId) {
- return new TaskExecutionFailedEventAck(taskInstanceId, true);
- }
-
- public static TaskExecutionFailedEventAck failed(int taskInstanceId) {
- return new TaskExecutionFailedEventAck(taskInstanceId, false);
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionKilledEventAck.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionKilledEventAck.java
deleted file mode 100644
index 38b6441c1f..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionKilledEventAck.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskExecutionKilledEventAck {
-
- private int taskInstanceId;
- private boolean success;
-
- public static TaskExecutionKilledEventAck success(int taskInstanceId) {
- return new TaskExecutionKilledEventAck(taskInstanceId, true);
- }
-
- public static TaskExecutionKilledEventAck failed(int taskInstanceId) {
- return new TaskExecutionKilledEventAck(taskInstanceId, false);
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionPausedEventAck.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionPausedEventAck.java
deleted file mode 100644
index fee22e9431..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionPausedEventAck.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskExecutionPausedEventAck {
-
- private int taskInstanceId;
- private boolean success;
-
- public static TaskExecutionPausedEventAck success(int taskInstanceId) {
- return new TaskExecutionPausedEventAck(taskInstanceId, true);
- }
-
- public static TaskExecutionPausedEventAck failed(int taskInstanceId) {
- return new TaskExecutionPausedEventAck(taskInstanceId, false);
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionSuccessEventAck.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionSuccessEventAck.java
deleted file mode 100644
index bcaba1d4fd..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskExecutionSuccessEventAck.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskExecutionSuccessEventAck {
-
- private int taskInstanceId;
- private boolean success;
-
- public static TaskExecutionSuccessEventAck success(int taskInstanceId) {
- return new TaskExecutionSuccessEventAck(taskInstanceId, true);
- }
-
- public static TaskExecutionSuccessEventAck failed(int taskInstanceId) {
- return new TaskExecutionSuccessEventAck(taskInstanceId, false);
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchResponse.java
deleted file mode 100644
index 8845ed9081..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceDispatchResponse.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskInstanceDispatchResponse {
-
- private Integer taskInstanceId;
-
- private boolean dispatchSuccess;
-
- private String message;
-
- public static TaskInstanceDispatchResponse success(Integer taskInstanceId) {
- return new TaskInstanceDispatchResponse(taskInstanceId, true, "dispatch success");
- }
-
- public static TaskInstanceDispatchResponse failed(Integer taskInstanceId, String message) {
- return new TaskInstanceDispatchResponse(taskInstanceId, false, message);
- }
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionDispatchedEventAck.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionDispatchedEventAck.java
deleted file mode 100644
index 088058284a..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionDispatchedEventAck.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskInstanceExecutionDispatchedEventAck {
-
- private int taskInstanceId;
- private boolean success;
-
- public static TaskInstanceExecutionDispatchedEventAck success(int taskInstanceId) {
- return new TaskInstanceExecutionDispatchedEventAck(taskInstanceId, true);
- }
-
- public static TaskInstanceExecutionDispatchedEventAck failed(int taskInstanceId) {
- return new TaskInstanceExecutionDispatchedEventAck(taskInstanceId, false);
- }
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionFinishEventAck.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionFinishEventAck.java
deleted file mode 100644
index 90ae49ab0e..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionFinishEventAck.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskInstanceExecutionFinishEventAck {
-
- private int taskInstanceId;
- private boolean success;
-
- public static TaskInstanceExecutionFinishEventAck success(int taskInstanceId) {
- return new TaskInstanceExecutionFinishEventAck(taskInstanceId, true);
- }
-
- public static TaskInstanceExecutionFinishEventAck failed(int taskInstanceId) {
- return new TaskInstanceExecutionFinishEventAck(taskInstanceId, false);
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionRunningEventAck.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionRunningEventAck.java
deleted file mode 100644
index aaeb4eb690..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceExecutionRunningEventAck.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskInstanceExecutionRunningEventAck {
-
- private int taskInstanceId;
- private boolean success;
-
- public static TaskInstanceExecutionRunningEventAck success(int taskInstanceId) {
- return new TaskInstanceExecutionRunningEventAck(taskInstanceId, true);
- }
-
- public static TaskInstanceExecutionRunningEventAck failed(int taskInstanceId) {
- return new TaskInstanceExecutionRunningEventAck(taskInstanceId, false);
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillResponse.java
deleted file mode 100644
index 2999a299bd..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillResponse.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.worker.transportor;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
-
-import java.util.Arrays;
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskInstanceKillResponse {
-
- private int taskInstanceId;
-
- private String host;
-
- private TaskExecutionStatus status;
-
- private int processId;
-
- /**
- * other resource manager appId , for example : YARN etc
- */
- private List<String> appIds;
-
- private boolean success;
-
- private String message;
-
- public static TaskInstanceKillResponse success(TaskExecutionContext taskExecutionContext) {
- TaskInstanceKillResponse taskInstanceKillResponse = new TaskInstanceKillResponse();
- taskInstanceKillResponse.setSuccess(true);
- taskInstanceKillResponse.setStatus(taskExecutionContext.getCurrentExecutionStatus());
- if (taskExecutionContext.getAppIds() != null) {
- taskInstanceKillResponse
- .setAppIds(Arrays.asList(taskExecutionContext.getAppIds().split(TaskConstants.COMMA)));
- }
- taskInstanceKillResponse.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
- taskInstanceKillResponse.setHost(taskExecutionContext.getHost());
- taskInstanceKillResponse.setProcessId(taskExecutionContext.getProcessId());
- return taskInstanceKillResponse;
- }
-
- public static TaskInstanceKillResponse fail(String message) {
- TaskInstanceKillResponse taskInstanceKillResponse = new TaskInstanceKillResponse();
- taskInstanceKillResponse.setSuccess(false);
- taskInstanceKillResponse.setMessage(message);
- return taskInstanceKillResponse;
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java
index 3d649fa8d8..c86e70268f 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointRequest.java
@@ -21,6 +21,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
+@Deprecated
@Data
@NoArgsConstructor
@AllArgsConstructor
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java
index 82093e1cf9..9d70b002f0 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceTriggerSavepointResponse.java
@@ -21,6 +21,7 @@ import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
+@Deprecated
@Data
@NoArgsConstructor
@AllArgsConstructor
diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml
index 81ca7171bc..8283e9e539 100644
--- a/dolphinscheduler-master/pom.xml
+++ b/dolphinscheduler-master/pom.xml
@@ -107,7 +107,11 @@
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-eventbus</artifactId>
- <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-task-executor</artifactId>
</dependency>
<dependency>
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 4cffd3e372..7695f7adc3 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -40,6 +40,7 @@ import org.apache.dolphinscheduler.server.master.engine.system.event.GlobalMaste
import org.apache.dolphinscheduler.server.master.metrics.MasterServerMetrics;
import org.apache.dolphinscheduler.server.master.registry.MasterRegistryClient;
import org.apache.dolphinscheduler.server.master.rpc.MasterRpcServer;
+import org.apache.dolphinscheduler.server.master.utils.MasterThreadFactory;
import org.apache.dolphinscheduler.service.ServiceConfiguration;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
@@ -166,6 +167,7 @@ public class MasterServer implements IStoppable {
}
// thread sleep 3 seconds for thread quietly stop
ThreadUtils.sleep(Constants.SERVER_CLOSE_WAIT_TIME.toMillis());
+ MasterThreadFactory.getDefaultSchedulerThreadExecutor().shutdownNow();
try (
SystemEventBusFireWorker systemEventBusFireWorker1 = systemEventBusFireWorker;
WorkflowEngine workflowEngine1 = workflowEngine;
@@ -176,7 +178,7 @@ public class MasterServer implements IStoppable {
// like ServerNodeManager,HostManager,TaskResponseService,CuratorZookeeperClient,etc
SpringApplicationContext closedSpringContext = springApplicationContext) {
- log.info("Master server is stopping, current cause : {}", cause);
+ log.info("MasterServer is stopping, current cause : {}", cause);
} catch (Exception e) {
log.error("MasterServer stop failed, current cause: {}", cause, e);
return;
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/LogicTaskConfig.java
similarity index 79%
copy from dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/LogicTaskConfig.java
index f4f8dddf12..46c5405ee3 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/LogicTaskConfig.java
@@ -15,17 +15,19 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.extract.worker.transportor;
+package org.apache.dolphinscheduler.server.master.config;
import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
-@NoArgsConstructor
+@Builder
@AllArgsConstructor
-public class TaskInstanceKillRequest {
-
- private Integer taskInstanceId;
+@NoArgsConstructor
+public class LogicTaskConfig {
+ @Builder.Default
+ private int taskExecutorThreadCount = Runtime.getRuntime().availableProcessors() * 2 + 1;
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
index bc1cfef2ec..76be92e36c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java
@@ -48,10 +48,8 @@ public class MasterConfig implements Validator {
private int workflowEventBusFireThreadCount = Runtime.getRuntime().availableProcessors() * 2 + 1;
- // todo: change to sync thread pool/ async thread pool ?
- private int masterSyncTaskExecutorThreadPoolSize = Runtime.getRuntime().availableProcessors();
+ private LogicTaskConfig logicTaskConfig = new LogicTaskConfig();
- private int masterAsyncTaskExecutorThreadPoolSize = Runtime.getRuntime().availableProcessors();
/**
* Master heart beat task execute interval.
*/
@@ -115,6 +113,7 @@ public class MasterConfig implements Validator {
"\n****************************Master Configuration**************************************" +
"\n listen-port -> " + listenPort +
"\n workflow-event-bus-fire-thread-count -> " + workflowEventBusFireThreadCount +
+ "\n logic-task-config -> " + logicTaskConfig +
"\n max-heartbeat-interval -> " + maxHeartbeatInterval +
"\n server-load-protection -> " + serverLoadProtection +
"\n master-address -> " + masterAddress +
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/IWorkflowRepository.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/IWorkflowRepository.java
index 026eda7e45..37084d3098 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/IWorkflowRepository.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/IWorkflowRepository.java
@@ -35,5 +35,4 @@ public interface IWorkflowRepository {
void remove(int workflowInstanceId);
- void clear();
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/TaskGroupCoordinator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/TaskGroupCoordinator.java
index 905a5d2fa8..19a20dff9a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/TaskGroupCoordinator.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/TaskGroupCoordinator.java
@@ -98,7 +98,7 @@ public class TaskGroupCoordinator extends BaseDaemonThread implements AutoClosea
private boolean flag = true;
- private static int DEFAULT_LIMIT = 1000;
+ private static final int DEFAULT_LIMIT = 1000;
public TaskGroupCoordinator() {
super("TaskGroupCoordinator");
@@ -126,7 +126,7 @@ public class TaskGroupCoordinator extends BaseDaemonThread implements AutoClosea
dealWithWaitingTaskGroupQueue();
taskGroupCoordinatorRoundCost.stop();
- log.info("TaskGroupCoordinator round cost: {}/ms", taskGroupCoordinatorRoundCost.getTime());
+ log.debug("TaskGroupCoordinator round cost: {}/ms", taskGroupCoordinatorRoundCost.getTime());
} finally {
registryClient.releaseLock(RegistryNodeType.MASTER_TASK_GROUP_COORDINATOR_LOCK.getRegistryPath());
}
@@ -183,7 +183,7 @@ public class TaskGroupCoordinator extends BaseDaemonThread implements AutoClosea
}
minTaskGroupQueueId = taskGroupQueues.get(taskGroupQueues.size() - 1).getId();
}
- log.info("Success amend TaskGroupQueue status cost: {}/ms", taskGroupCoordinatorRoundTimeCost.getTime());
+ log.debug("Success amend TaskGroupQueue status cost: {}/ms", taskGroupCoordinatorRoundTimeCost.getTime());
}
/**
@@ -237,7 +237,7 @@ public class TaskGroupCoordinator extends BaseDaemonThread implements AutoClosea
}
minTaskGroupQueueId = taskGroupQueues.get(taskGroupQueues.size() - 1).getId();
}
- log.info("Success deal with force start TaskGroupQueue cost: {}/ms",
+ log.debug("Success deal with force start TaskGroupQueue cost: {}/ms",
taskGroupCoordinatorRoundTimeCost.getTime());
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowCacheRepository.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowCacheRepository.java
index 85d01fb51e..6f069f8eee 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowCacheRepository.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowCacheRepository.java
@@ -68,8 +68,4 @@ public class WorkflowCacheRepository implements IWorkflowRepository {
return ImmutableList.copyOf(workflowExecutionRunnableMap.values());
}
- @Override
- public void clear() {
- workflowExecutionRunnableMap.clear();
- }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEngine.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEngine.java
index 76ca89dd57..75bc6b4527 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEngine.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEngine.java
@@ -18,7 +18,8 @@
package org.apache.dolphinscheduler.server.master.engine;
import org.apache.dolphinscheduler.server.master.engine.command.CommandEngine;
-import org.apache.dolphinscheduler.server.master.runner.MasterTaskExecutorBootstrap;
+import org.apache.dolphinscheduler.server.master.engine.executor.LogicTaskEngineDelegator;
+import org.apache.dolphinscheduler.server.master.runner.GlobalTaskDispatchWaitingQueueLooper;
import lombok.extern.slf4j.Slf4j;
@@ -36,31 +37,38 @@ public class WorkflowEngine implements AutoCloseable {
private WorkflowEventBusCoordinator workflowEventBusCoordinator;
@Autowired
- private MasterTaskExecutorBootstrap masterTaskExecutorBootstrap;
+ private CommandEngine commandEngine;
@Autowired
- private CommandEngine commandEngine;
+ private GlobalTaskDispatchWaitingQueueLooper globalTaskDispatchWaitingQueueLooper;
+
+ @Autowired
+ private LogicTaskEngineDelegator logicTaskEngineDelegator;
public void start() {
taskGroupCoordinator.start();
- masterTaskExecutorBootstrap.start();
-
workflowEventBusCoordinator.start();
commandEngine.start();
+ globalTaskDispatchWaitingQueueLooper.start();
+
+ logicTaskEngineDelegator.start();
+
log.info("WorkflowEngine started");
}
@Override
public void close() throws Exception {
try (
- final CommandEngine commandEngine1 = commandEngine;
- final WorkflowEventBusCoordinator workflowEventBusCoordinator1 = workflowEventBusCoordinator;
- final MasterTaskExecutorBootstrap masterTaskExecutorBootstrap1 = masterTaskExecutorBootstrap;
- final TaskGroupCoordinator taskGroupCoordinator1 = taskGroupCoordinator) {
+ final CommandEngine ignore1 = commandEngine;
+ final WorkflowEventBusCoordinator ignore2 = workflowEventBusCoordinator;
+ final GlobalTaskDispatchWaitingQueueLooper ignore3 =
+ globalTaskDispatchWaitingQueueLooper;
+ final TaskGroupCoordinator ignore4 = taskGroupCoordinator;
+ final LogicTaskEngineDelegator ignore5 = logicTaskEngineDelegator) {
// closed the resource
}
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java
index 5191d32a51..905f4b7842 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorker.java
@@ -78,7 +78,7 @@ public class WorkflowEventBusFireWorker {
if (CollectionUtils.isEmpty(workflowExecutionRunnables)) {
return;
}
- for (IWorkflowExecutionRunnable workflowExecutionRunnable : workflowExecutionRunnables) {
+ for (final IWorkflowExecutionRunnable workflowExecutionRunnable : workflowExecutionRunnables) {
final Integer workflowInstanceId = workflowExecutionRunnable.getId();
final String workflowInstanceName = workflowExecutionRunnable.getName();
try {
@@ -92,6 +92,10 @@ public class WorkflowEventBusFireWorker {
}
}
+ public int getRegisteredWorkflowExecuteRunnableSize() {
+ return registeredWorkflowExecuteRunnableMap.size();
+ }
+
private List<IWorkflowExecutionRunnable> getWaitingFireWorkflowExecutionRunnables() {
if (MapUtils.isEmpty(registeredWorkflowExecuteRunnableMap)) {
return Collections.emptyList();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorkers.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorkers.java
index 5542e7ae4d..fdc038eb32 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorkers.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/WorkflowEventBusFireWorkers.java
@@ -30,6 +30,8 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+import com.google.common.annotations.VisibleForTesting;
+
@Slf4j
@Component
public class WorkflowEventBusFireWorkers implements AutoCloseable {
@@ -50,7 +52,7 @@ public class WorkflowEventBusFireWorkers implements AutoCloseable {
final int workflowEventBusFireThreadCount = masterConfig.getWorkflowEventBusFireThreadCount();
workflowEventBusFireThreadPool = Executors.newScheduledThreadPool(
workflowEventBusFireThreadCount,
- ThreadUtils.newDaemonThreadFactory("DS-WorkflowEventBusFireWorker-%d"));
+ ThreadUtils.newDaemonThreadFactory("ds-workflow-eventbus-worker-%d"));
workflowEventBusFireWorkers = new WorkflowEventBusFireWorker[workflowEventBusFireThreadCount];
for (int i = 0; i < workflowEventBusFireThreadCount; i++) {
@@ -73,6 +75,11 @@ public class WorkflowEventBusFireWorkers implements AutoCloseable {
return workflowEventBusFireWorkers[workerSlot];
}
+ @VisibleForTesting
+ public WorkflowEventBusFireWorker[] getWorkers() {
+ return workflowEventBusFireWorkers;
+ }
+
public int getWorkerSize() {
return masterConfig.getWorkflowEventBusFireThreadCount();
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/IdSlotBasedCommandFetcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/IdSlotBasedCommandFetcher.java
index cfd13b4e45..f76a0dfa47 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/IdSlotBasedCommandFetcher.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/command/IdSlotBasedCommandFetcher.java
@@ -64,7 +64,7 @@ public class IdSlotBasedCommandFetcher implements ICommandFetcher {
idSlotBasedFetchConfig.getIdStep(),
idSlotBasedFetchConfig.getFetchSize());
long cost = System.currentTimeMillis() - scheduleStartTime;
- log.info("[Slot-{}/{}] Fetch {} commands in {}ms.", currentSlotIndex, totalSlot, commands.size(), cost);
+ log.debug("[Slot-{}/{}] Fetch {} commands in {}ms.", currentSlotIndex, totalSlot, commands.size(), cost);
WorkflowInstanceMetrics.recordCommandQueryTime(cost);
return commands;
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/exceptions/TaskReassignMasterHostException.java
similarity index 69%
copy from dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/exceptions/TaskReassignMasterHostException.java
index f4f8dddf12..0e44e7af64 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-worker/src/main/java/org/apache/dolphinscheduler/extract/worker/transportor/TaskInstanceKillRequest.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/exceptions/TaskReassignMasterHostException.java
@@ -15,17 +15,16 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.extract.worker.transportor;
+package org.apache.dolphinscheduler.server.master.engine.exceptions;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
+public class TaskReassignMasterHostException extends RuntimeException {
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskInstanceKillRequest {
+ public TaskReassignMasterHostException(String message) {
+ super(message);
+ }
- private Integer taskInstanceId;
+ public TaskReassignMasterHostException(String message, Throwable cause) {
+ super(message, cause);
+ }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskEngineDelegator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskEngineDelegator.java
new file mode 100644
index 0000000000..da76deec16
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskEngineDelegator.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.engine.executor;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
+import org.apache.dolphinscheduler.task.executor.TaskEngine;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+public class LogicTaskEngineDelegator implements AutoCloseable {
+
+ private final TaskEngine taskEngine;
+
+ private final LogicTaskExecutorFactory logicTaskExecutorFactory;
+
+ private final LogicTaskExecutorLifecycleEventReporter logicTaskExecutorEventReporter;
+
+ public LogicTaskEngineDelegator(final LogicTaskEngineFactory logicTaskEngineFactory,
+ final LogicTaskExecutorFactory logicTaskExecutorFactory,
+ final LogicTaskExecutorLifecycleEventReporter logicTaskExecutorEventReporter) {
+ this.logicTaskExecutorFactory = logicTaskExecutorFactory;
+ this.taskEngine = logicTaskEngineFactory.createTaskEngine();
+ this.logicTaskExecutorEventReporter = logicTaskExecutorEventReporter;
+ }
+
+ public void start() {
+ taskEngine.start();
+ logicTaskExecutorEventReporter.start();
+ log.info("LogicTaskEngineDelegator started");
+ }
+
+ public void dispatchLogicTask(final TaskExecutionContext taskExecutionContext) {
+ final ITaskExecutor taskExecutor = logicTaskExecutorFactory.createTaskExecutor(taskExecutionContext);
+ taskEngine.submitTask(taskExecutor);
+ }
+
+ public void killLogicTask(final int taskInstanceId) {
+ taskEngine.killTask(taskInstanceId);
+ }
+
+ public void pauseLogicTask(final int taskInstanceId) {
+ taskEngine.pauseTask(taskInstanceId);
+ }
+
+ public void ackLogicTaskExecutionEvent(final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck) {
+ logicTaskExecutorEventReporter.receiveTaskExecutorLifecycleEventACK(taskExecutorLifecycleEventAck);
+ }
+
+ @Override
+ public void close() {
+ try (
+ final TaskEngine ignore1 = taskEngine;
+ final LogicTaskExecutorLifecycleEventReporter ignore2 = logicTaskExecutorEventReporter) {
+ log.info("LogicTaskEngineDelegator closed");
+ }
+ }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskEngineFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskEngineFactory.java
new file mode 100644
index 0000000000..d098946bbb
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskEngineFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.engine.executor;
+
+import org.apache.dolphinscheduler.task.executor.TaskEngine;
+import org.apache.dolphinscheduler.task.executor.TaskEngineBuilder;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class LogicTaskEngineFactory {
+
+ private final LogicTaskExecutorRepository logicTaskExecutorRepository;
+
+ private final LogicTaskExecutorContainerProvider logicTaskExecutorContainerDelegator;
+
+ private final LogicTaskExecutorEventBusCoordinator logicTaskExecutorEventBusCoordinator;
+
+ public LogicTaskEngineFactory(final LogicTaskExecutorRepository logicTaskExecutorRepository,
+ final LogicTaskExecutorContainerProvider logicTaskExecutorContainerDelegator,
+ final LogicTaskExecutorEventBusCoordinator logicTaskExecutorEventBusCoordinator) {
+ this.logicTaskExecutorRepository = logicTaskExecutorRepository;
+ this.logicTaskExecutorContainerDelegator = logicTaskExecutorContainerDelegator;
+ this.logicTaskExecutorEventBusCoordinator = logicTaskExecutorEventBusCoordinator;
+ }
+
+ public TaskEngine createTaskEngine() {
+ final TaskEngineBuilder taskEngineBuilder = TaskEngineBuilder.builder()
+ .engineName("LogicTaskEngine")
+ .taskExecutorRepository(logicTaskExecutorRepository)
+ .taskExecutorContainerDelegator(logicTaskExecutorContainerDelegator)
+ .taskExecutorEventBusCoordinator(logicTaskExecutorEventBusCoordinator)
+ .build();
+ return new TaskEngine(taskEngineBuilder);
+ }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutor.java
new file mode 100644
index 0000000000..4dafabfe4d
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutor.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.engine.executor;
+
+import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ILogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.LogicTaskPluginFactoryBuilder;
+import org.apache.dolphinscheduler.task.executor.AbstractTaskExecutor;
+import org.apache.dolphinscheduler.task.executor.TaskExecutorState;
+import org.apache.dolphinscheduler.task.executor.TaskExecutorStateMappings;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class LogicTaskExecutor extends AbstractTaskExecutor {
+
+ private ILogicTask<? extends AbstractParameters> logicTask;
+
+ private final LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder;
+
+ public LogicTaskExecutor(final LogicTaskExecutorBuilder logicTaskExecutorBuilder) {
+ super(logicTaskExecutorBuilder.getTaskExecutionContext(), logicTaskExecutorBuilder.getTaskExecutorEventBus());
+ this.logicTaskPluginFactoryBuilder = logicTaskExecutorBuilder.getLogicTaskPluginFactoryBuilder();
+ }
+
+ @Override
+ protected TaskExecutorState doTrackTaskPluginStatus() {
+ return TaskExecutorStateMappings.mapState(logicTask.getTaskExecutionState());
+ }
+
+ @SneakyThrows
+ @Override
+ protected void initializeTaskPlugin() {
+ logicTask = logicTaskPluginFactoryBuilder
+ .createILogicTaskPluginFactory(taskExecutionContext.getTaskType())
+ .createLogicTask(this);
+ log.info("Initialized task plugin instance: {} successfully", taskExecutionContext.getTaskType());
+ }
+
+ @SneakyThrows
+ @Override
+ protected void doTriggerTaskPlugin() {
+ logicTask.start();
+ }
+
+ @SneakyThrows
+ @Override
+ public void pause() {
+ // todo: judge the status, whether the task is running, we should support to pause the task which is not running
+ // if the status is initialized, then we can directly change to paused
+ if (logicTask != null) {
+ logicTask.pause();
+ }
+ }
+
+ @SneakyThrows
+ @Override
+ public void kill() {
+ if (logicTask != null) {
+ logicTask.kill();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LogicTaskExecutor{" +
+ "id=" + taskExecutionContext.getTaskInstanceId() +
+ ", name=" + taskExecutionContext.getTaskName() +
+ ", stat=" + taskExecutorState.get() +
+ '}';
+ }
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskExecutionKilledEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorBuilder.java
similarity index 60%
rename from dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskExecutionKilledEvent.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorBuilder.java
index 527f019c75..4d245a164d 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskExecutionKilledEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorBuilder.java
@@ -15,35 +15,26 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.extract.master.transportor;
+package org.apache.dolphinscheduler.server.master.engine.executor;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.LogicTaskPluginFactoryBuilder;
+import org.apache.dolphinscheduler.task.executor.eventbus.TaskExecutorEventBus;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
-import lombok.NoArgsConstructor;
@Data
@Builder
-@NoArgsConstructor
@AllArgsConstructor
-public class TaskExecutionKilledEvent implements ITaskExecutionEvent {
-
- private int taskInstanceId;
-
- private int workflowInstanceId;
-
- private String taskInstanceHost;
-
- private String workflowInstanceHost;
+public class LogicTaskExecutorBuilder {
- private long endTime;
+ private TaskExecutionContext taskExecutionContext;
- private long eventCreateTime;
+ @Builder.Default
+ private TaskExecutorEventBus taskExecutorEventBus = new TaskExecutorEventBus();
- private long eventSendTime;
+ private LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder;
- @Override
- public TaskInstanceExecutionEventType getEventType() {
- return TaskInstanceExecutionEventType.KILLED;
- }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorContainerProvider.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorContainerProvider.java
new file mode 100644
index 0000000000..709f9d61df
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorContainerProvider.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.engine.executor;
+
+import org.apache.dolphinscheduler.server.master.config.MasterConfig;
+import org.apache.dolphinscheduler.task.executor.container.ITaskExecutorContainer;
+import org.apache.dolphinscheduler.task.executor.container.ITaskExecutorContainerProvider;
+import org.apache.dolphinscheduler.task.executor.container.SharedThreadTaskExecutorContainer;
+import org.apache.dolphinscheduler.task.executor.container.TaskExecutorContainerConfig;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class LogicTaskExecutorContainerProvider implements ITaskExecutorContainerProvider {
+
+ private final ITaskExecutorContainer taskExecutorContainer;
+
+ public LogicTaskExecutorContainerProvider(final MasterConfig masterConfig) {
+ final TaskExecutorContainerConfig containerConfig = TaskExecutorContainerConfig.builder()
+ .taskExecutorThreadPoolSize(masterConfig.getLogicTaskConfig().getTaskExecutorThreadCount())
+ .containerName("shared-task-executor-container")
+ .build();
+ this.taskExecutorContainer = new SharedThreadTaskExecutorContainer(containerConfig);
+ }
+
+ @Override
+ public ITaskExecutorContainer getExecutorContainer() {
+ return taskExecutorContainer;
+ }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueue.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorEventBusCoordinator.java
similarity index 53%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueue.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorEventBusCoordinator.java
index 51bda7edbe..d696633cba 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueue.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorEventBusCoordinator.java
@@ -15,32 +15,18 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.execute;
+package org.apache.dolphinscheduler.server.master.engine.executor;
-import java.util.concurrent.DelayQueue;
-
-import javax.annotation.Nullable;
-
-import lombok.NonNull;
+import org.apache.dolphinscheduler.task.executor.eventbus.TaskExecutorEventBusCoordinator;
import org.springframework.stereotype.Component;
@Component
-public class AsyncMasterTaskDelayQueue {
-
- private final DelayQueue<AsyncTaskExecutionContext> asyncTaskCheckDelayQueue = new DelayQueue<>();
-
- public void addAsyncTask(@NonNull AsyncTaskExecutionContext asyncTaskExecutionContext) {
- asyncTaskExecutionContext.refreshStartTime();
- asyncTaskCheckDelayQueue.add(asyncTaskExecutionContext);
- }
-
- public @Nullable AsyncTaskExecutionContext pollAsyncTask() throws InterruptedException {
- return asyncTaskCheckDelayQueue.take();
- }
+public class LogicTaskExecutorEventBusCoordinator extends TaskExecutorEventBusCoordinator {
- public int getAsyncTaskRunningNum() {
- return asyncTaskCheckDelayQueue.size();
+ public LogicTaskExecutorEventBusCoordinator(final LogicTaskExecutorRepository logicTaskExecutorRepository,
+ final LogicTaskExecutorLifecycleEventListener logicTaskExecutorLifecycleEventListener) {
+ super("LogicTaskExecutorEventBusCoordinator", logicTaskExecutorRepository);
+ registerTaskExecutorLifecycleEventListener(logicTaskExecutorLifecycleEventListener);
}
-
}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorEventRemoteReporterClient.java
similarity index 71%
copy from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorEventRemoteReporterClient.java
index e8865c985c..4cb7551ba9 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorEventRemoteReporterClient.java
@@ -15,10 +15,13 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.worker.runner.operator;
+package org.apache.dolphinscheduler.server.master.engine.executor;
-public interface ITaskInstanceOperationFunction<X, Y> {
+import org.apache.dolphinscheduler.extract.master.TaskExecutorEventRemoteReporterClient;
- Y operate(X x);
+import org.springframework.stereotype.Component;
+
+@Component
+public class LogicTaskExecutorEventRemoteReporterClient extends TaskExecutorEventRemoteReporterClient {
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorFactory.java
new file mode 100644
index 0000000000..355fdf76bc
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.engine.executor;
+
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.LogicTaskPluginFactoryBuilder;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutorFactory;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+public class LogicTaskExecutorFactory implements ITaskExecutorFactory {
+
+ private final LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder;
+
+ public LogicTaskExecutorFactory(final LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder) {
+ this.logicTaskPluginFactoryBuilder = logicTaskPluginFactoryBuilder;
+ }
+
+ @Override
+ public ITaskExecutor createTaskExecutor(final TaskExecutionContext taskExecutionContext) {
+ assemblyTaskLogPath(taskExecutionContext);
+
+ final LogicTaskExecutorBuilder logicTaskExecutorBuilder = LogicTaskExecutorBuilder.builder()
+ .taskExecutionContext(taskExecutionContext)
+ .logicTaskPluginFactoryBuilder(logicTaskPluginFactoryBuilder)
+ .build();
+ return new LogicTaskExecutor(logicTaskExecutorBuilder);
+ }
+
+ private void assemblyTaskLogPath(final TaskExecutionContext taskExecutionContext) {
+ taskExecutionContext.setLogPath(LogUtils.getTaskInstanceLogFullPath(taskExecutionContext));
+ }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorLifecycleEventListener.java
similarity index 50%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorLifecycleEventListener.java
index 9943ee1b5b..e9efc60a2d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatchFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorLifecycleEventListener.java
@@ -15,35 +15,22 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.dispatcher;
+package org.apache.dolphinscheduler.server.master.engine.executor;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
+import org.apache.dolphinscheduler.task.executor.listener.TaskExecutorLifecycleEventListener;
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-@Slf4j
@Component
-public class TaskDispatchFactory {
-
- @Autowired
- private MasterTaskDispatcher masterTaskDispatcher;
-
- @Autowired
- private WorkerTaskDispatcher workerTaskDispatcher;
-
- public TaskDispatcher getTaskDispatcher(String taskType) {
- if (TaskTypeUtils.isLogicTask(taskType)) {
- return masterTaskDispatcher;
- }
- return workerTaskDispatcher;
+public class LogicTaskExecutorLifecycleEventListener extends TaskExecutorLifecycleEventListener {
+
+ public LogicTaskExecutorLifecycleEventListener(
+ final LogicTaskExecutorContainerProvider logicTaskExecutorContainerDelegator,
+ final LogicTaskExecutorRepository logicTaskExecutorRepository,
+ final LogicTaskExecutorLifecycleEventReporter logicTaskExecutorEventReporter) {
+ super(
+ logicTaskExecutorContainerDelegator,
+ logicTaskExecutorRepository,
+ logicTaskExecutorEventReporter);
}
-
- public TaskDispatcher getTaskDispatcher(TaskInstance taskInstance) {
- return getTaskDispatcher(taskInstance.getTaskType());
- }
-
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseResponse.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorLifecycleEventReporter.java
similarity index 56%
copy from dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseResponse.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorLifecycleEventReporter.java
index b91c275074..41a80ba716 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskPauseResponse.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorLifecycleEventReporter.java
@@ -15,25 +15,17 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.extract.master.transportor;
+package org.apache.dolphinscheduler.server.master.engine.executor;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
+import org.apache.dolphinscheduler.task.executor.eventbus.TaskExecutorLifecycleEventRemoteReporter;
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class LogicTaskPauseResponse {
+import org.springframework.stereotype.Component;
- private boolean success;
- private String message;
+@Component
+public class LogicTaskExecutorLifecycleEventReporter extends TaskExecutorLifecycleEventRemoteReporter {
- public static LogicTaskPauseResponse success() {
- return new LogicTaskPauseResponse(true, null);
- }
-
- public static LogicTaskPauseResponse fail(String message) {
- return new LogicTaskPauseResponse(false, message);
+ public LogicTaskExecutorLifecycleEventReporter(
+ final LogicTaskExecutorEventRemoteReporterClient logicTaskExecutorEventRemoteReporterClient) {
+ super("LogicTaskExecutorLifecycleEventReporter", logicTaskExecutorEventRemoteReporterClient);
}
}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorRepository.java
similarity index 74%
copy from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorRepository.java
index e8865c985c..510ad88f19 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/LogicTaskExecutorRepository.java
@@ -15,10 +15,13 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.worker.runner.operator;
+package org.apache.dolphinscheduler.server.master.engine.executor;
-public interface ITaskInstanceOperationFunction<X, Y> {
+import org.apache.dolphinscheduler.task.executor.TaskExecutorRepository;
- Y operate(X x);
+import org.springframework.stereotype.Component;
+
+@Component
+public class LogicTaskExecutorRepository extends TaskExecutorRepository {
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/AbstractLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/AbstractLogicTask.java
new file mode 100644
index 0000000000..98f78aa452
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/AbstractLogicTask.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin;
+
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
+
+import lombok.extern.slf4j.Slf4j;
+
+import com.google.common.base.Preconditions;
+
+@Slf4j
+public abstract class AbstractLogicTask<T extends AbstractParameters> implements ILogicTask<T> {
+
+ protected final TaskExecutionContext taskExecutionContext;
+
+ protected final T taskParameters;
+
+ protected TaskExecutionStatus taskExecutionStatus;
+
+ public AbstractLogicTask(final TaskExecutionContext taskExecutionContext) {
+ this.taskExecutionContext = taskExecutionContext;
+ this.taskParameters = getTaskParameterDeserializer().deserialize(taskExecutionContext.getTaskParams());
+ Preconditions.checkNotNull(taskParameters,
+ "Deserialize task parameters: " + taskExecutionContext.getTaskParams());
+ log.info("Success initialize parameters: \n{}", JSONUtils.toPrettyJsonString(taskParameters));
+ }
+
+ @Override
+ public TaskExecutionStatus getTaskExecutionState() {
+ return taskExecutionStatus;
+ }
+
+ protected boolean isRunning() {
+ return taskExecutionStatus == TaskExecutionStatus.RUNNING_EXECUTION;
+ }
+
+ protected void onTaskRunning() {
+ taskExecutionStatus = TaskExecutionStatus.RUNNING_EXECUTION;
+ }
+
+ protected void onTaskSuccess() {
+ taskExecutionStatus = TaskExecutionStatus.SUCCESS;
+ }
+
+ protected void onTaskFailed() {
+ taskExecutionStatus = TaskExecutionStatus.FAILURE;
+ }
+
+ protected void onTaskKilled() {
+ taskExecutionStatus = TaskExecutionStatus.KILL;
+ }
+
+ protected void onTaskPaused() {
+ taskExecutionStatus = TaskExecutionStatus.PAUSE;
+ }
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ILogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ILogicTask.java
similarity index 74%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ILogicTask.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ILogicTask.java
index 806792f45a..5052b0b73c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ILogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ILogicTask.java
@@ -15,20 +15,22 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-public interface ILogicTask {
+public interface ILogicTask<T extends AbstractParameters> {
+
+ void start() throws MasterTaskExecuteException;
void pause() throws MasterTaskExecuteException;
void kill() throws MasterTaskExecuteException;
- AbstractParameters getTaskParameters();
+ TaskExecutionStatus getTaskExecutionState();
- TaskExecutionContext getTaskExecutionContext();
+ ITaskParameterDeserializer<T> getTaskParameterDeserializer();
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ILogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ILogicTaskPluginFactory.java
similarity index 69%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ILogicTaskPluginFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ILogicTaskPluginFactory.java
index 08b2285184..7beacc204f 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ILogicTaskPluginFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ILogicTaskPluginFactory.java
@@ -15,14 +15,15 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
-public interface ILogicTaskPluginFactory<T extends ILogicTask> {
+public interface ILogicTaskPluginFactory<T extends ILogicTask<? extends AbstractParameters>> {
- T createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException;
+ T createLogicTask(final ITaskExecutor taskExecutor) throws LogicTaskInitializeException;
String getTaskType();
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ITaskParameterDeserializer.java
similarity index 83%
rename from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ITaskParameterDeserializer.java
index e8865c985c..31d4aa5bf0 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/operator/ITaskInstanceOperationFunction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/ITaskParameterDeserializer.java
@@ -15,10 +15,10 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.worker.runner.operator;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin;
-public interface ITaskInstanceOperationFunction<X, Y> {
+public interface ITaskParameterDeserializer<T> {
- Y operate(X x);
+ T deserialize(String taskParamsJson);
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/LogicTaskPluginFactoryBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/LogicTaskPluginFactoryBuilder.java
similarity index 68%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/LogicTaskPluginFactoryBuilder.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/LogicTaskPluginFactoryBuilder.java
index 7d6cc7260c..5642e407aa 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/LogicTaskPluginFactoryBuilder.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/LogicTaskPluginFactoryBuilder.java
@@ -15,8 +15,9 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskFactoryNotFoundException;
import java.util.List;
@@ -29,19 +30,20 @@ import org.springframework.stereotype.Component;
@Slf4j
@Component
-@SuppressWarnings("unchecked")
public class LogicTaskPluginFactoryBuilder {
- private final Map<String, ILogicTaskPluginFactory> logicTaskPluginFactoryMap = new ConcurrentHashMap<>();
+ private final Map<String, ILogicTaskPluginFactory<? extends ILogicTask<? extends AbstractParameters>>> logicTaskPluginFactoryMap =
+ new ConcurrentHashMap<>();
- public LogicTaskPluginFactoryBuilder(List<ILogicTaskPluginFactory> logicTaskPluginFactories) {
+ public LogicTaskPluginFactoryBuilder(List<ILogicTaskPluginFactory<? extends ILogicTask<? extends AbstractParameters>>> logicTaskPluginFactories) {
logicTaskPluginFactories.forEach(
logicTaskPluginFactory -> logicTaskPluginFactoryMap.put(logicTaskPluginFactory.getTaskType(),
logicTaskPluginFactory));
}
- public ILogicTaskPluginFactory createILogicTaskPluginFactory(String taskType) throws LogicTaskFactoryNotFoundException {
- ILogicTaskPluginFactory logicTaskPluginFactory = logicTaskPluginFactoryMap.get(taskType);
+ public ILogicTaskPluginFactory<? extends ILogicTask<? extends AbstractParameters>> createILogicTaskPluginFactory(String taskType) throws LogicTaskFactoryNotFoundException {
+ ILogicTaskPluginFactory<? extends ILogicTask<? extends AbstractParameters>> logicTaskPluginFactory =
+ logicTaskPluginFactoryMap.get(taskType);
if (logicTaskPluginFactory == null) {
throw new LogicTaskFactoryNotFoundException("Cannot find the logic task factory: " + taskType);
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/condition/ConditionLogicTask.java
similarity index 76%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/condition/ConditionLogicTask.java
index 6088ff53ea..dcfca074e4 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/condition/ConditionLogicTask.java
@@ -15,19 +15,20 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.condition;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.condition;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.AbstractLogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ITaskParameterDeserializer;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
+import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
import java.util.List;
import java.util.Map;
@@ -40,28 +41,35 @@ import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j
-public class ConditionLogicTask extends BaseSyncLogicTask<ConditionsParameters> {
-
- public static final String TASK_TYPE = "CONDITIONS";
+public class ConditionLogicTask extends AbstractLogicTask<ConditionsParameters> {
private final TaskInstanceDao taskInstanceDao;
+ private final TaskInstance taskInstance;
+
public ConditionLogicTask(IWorkflowExecutionRunnable workflowExecutionRunnable,
TaskExecutionContext taskExecutionContext,
TaskInstanceDao taskInstanceDao) {
- super(workflowExecutionRunnable, taskExecutionContext,
- JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<ConditionsParameters>() {
- }));
+ super(taskExecutionContext);
+ this.taskInstance = workflowExecutionRunnable
+ .getWorkflowExecuteContext()
+ .getWorkflowExecutionGraph()
+ .getTaskExecutionRunnableById(taskExecutionContext.getTaskInstanceId())
+ .getTaskInstance();
this.taskInstanceDao = taskInstanceDao;
+ onTaskRunning();
}
@Override
- public void handle() {
+ public void start() {
+
DependResult conditionResult = calculateConditionResult();
log.info("The condition result is {}", conditionResult);
taskParameters.getConditionResult().setConditionSuccess(conditionResult == DependResult.SUCCESS);
+
taskInstance.setTaskParams(JSONUtils.toJsonString(taskParameters));
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
+
+ onTaskSuccess();
}
private DependResult calculateConditionResult() {
@@ -100,4 +108,21 @@ public class ConditionLogicTask extends BaseSyncLogicTask<ConditionsParameters>
log.info("The dependent result will be: {}", dependResult);
return dependResult;
}
+
+ @Override
+ public void pause() throws MasterTaskExecuteException {
+ log.info("The ConditionTask does not support pause operation");
+ }
+
+ @Override
+ public void kill() throws MasterTaskExecuteException {
+ log.info("The ConditionTask does not support kill operation");
+ }
+
+ @Override
+ public ITaskParameterDeserializer<ConditionsParameters> getTaskParameterDeserializer() {
+ return taskParamsJson -> JSONUtils.parseObject(taskParamsJson, new TypeReference<ConditionsParameters>() {
+ });
+ }
+
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/condition/ConditionLogicTaskPluginFactory.java
similarity index 80%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/condition/ConditionLogicTaskPluginFactory.java
index dc797c6a68..7c4ee34895 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/condition/ConditionLogicTaskPluginFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/condition/ConditionLogicTaskPluginFactory.java
@@ -15,14 +15,15 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.condition;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.condition;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.task.ConditionsLogicTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ILogicTaskPluginFactory;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
import lombok.extern.slf4j.Slf4j;
@@ -40,8 +41,9 @@ public class ConditionLogicTaskPluginFactory implements ILogicTaskPluginFactory<
private IWorkflowRepository workflowExecutionRunnableMemoryRepository;
@Override
- public ConditionLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) {
- IWorkflowExecutionRunnable workflowExecutionRunnable =
+ public ConditionLogicTask createLogicTask(final ITaskExecutor taskExecutor) {
+ final TaskExecutionContext taskExecutionContext = taskExecutor.getTaskExecutionContext();
+ final IWorkflowExecutionRunnable workflowExecutionRunnable =
workflowExecutionRunnableMemoryRepository.get(taskExecutionContext.getWorkflowInstanceId());
return new ConditionLogicTask(workflowExecutionRunnable, taskExecutionContext, taskInstanceDao);
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentLogicTask.java
similarity index 60%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentLogicTask.java
index 70ac2ba10b..3e4d7314f0 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentLogicTask.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dependent;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dependent;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.repository.ProjectDao;
@@ -24,30 +24,23 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.AbstractLogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ITaskParameterDeserializer;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
-import org.apache.dolphinscheduler.server.master.runner.task.BaseAsyncLogicTask;
import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j
-public class DependentLogicTask extends BaseAsyncLogicTask<DependentParameters> {
+public class DependentLogicTask extends AbstractLogicTask<DependentParameters> {
- public static final String TASK_TYPE = "DEPENDENT";
+ private final TaskExecutionContext taskExecutionContext;
- private final ProjectDao projectDao;
- private final WorkflowDefinitionDao workflowDefinitionDao;
- private final TaskDefinitionDao taskDefinitionDao;
- private final TaskInstanceDao taskInstanceDao;
- private final WorkflowInstanceDao workflowInstanceDao;
-
- private final IWorkflowExecutionRunnable workflowExecutionRunnable;
-
- private DependentAsyncTaskExecuteFunction dependentAsyncTaskExecuteFunction;
+ private final DependentTaskTracker dependentTaskTracker;
public DependentLogicTask(TaskExecutionContext taskExecutionContext,
ProjectDao projectDao,
@@ -56,38 +49,49 @@ public class DependentLogicTask extends BaseAsyncLogicTask<DependentParameters>
TaskInstanceDao taskInstanceDao,
WorkflowInstanceDao workflowInstanceDao,
IWorkflowExecutionRunnable workflowExecutionRunnable) {
- super(taskExecutionContext,
- JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<DependentParameters>() {
- }));
- this.projectDao = projectDao;
- this.workflowDefinitionDao = workflowDefinitionDao;
- this.taskDefinitionDao = taskDefinitionDao;
- this.taskInstanceDao = taskInstanceDao;
- this.workflowInstanceDao = workflowInstanceDao;
- this.workflowExecutionRunnable = workflowExecutionRunnable;
-
- }
-
- @Override
- public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() {
- dependentAsyncTaskExecuteFunction = new DependentAsyncTaskExecuteFunction(taskExecutionContext,
+ super(taskExecutionContext);
+ this.taskExecutionContext = taskExecutionContext;
+ this.dependentTaskTracker = new DependentTaskTracker(
+ taskExecutionContext,
taskParameters,
projectDao,
workflowDefinitionDao,
taskDefinitionDao,
taskInstanceDao,
workflowInstanceDao);
- return dependentAsyncTaskExecuteFunction;
+ onTaskRunning();
+ }
+
+ @Override
+ public void start() throws MasterTaskExecuteException {
+ log.info("Dependent task: {} started", taskExecutionContext.getTaskName());
+ }
+
+ @Override
+ public TaskExecutionStatus getTaskExecutionState() {
+ if (isRunning()) {
+ taskExecutionStatus = dependentTaskTracker.getDependentTaskStatus();
+ return taskExecutionStatus;
+ }
+ return taskExecutionStatus;
}
@Override
public void pause() throws MasterTaskExecuteException {
- // todo: support pause
+ onTaskPaused();
+ log.info("Pause task : {} success", taskExecutionContext.getTaskName());
}
@Override
public void kill() throws MasterTaskExecuteException {
- // todo: support kill
+ onTaskKilled();
+ log.info("Kill task : {} success", taskExecutionContext.getTaskName());
+ }
+
+ @Override
+ public ITaskParameterDeserializer<DependentParameters> getTaskParameterDeserializer() {
+ return taskParamsJson -> JSONUtils.parseObject(taskParamsJson, new TypeReference<DependentParameters>() {
+ });
}
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentLogicTaskPluginFactory.java
similarity index 82%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentLogicTaskPluginFactory.java
index 31c4f67bf6..6c4f35b1fb 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentLogicTaskPluginFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentLogicTaskPluginFactory.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dependent;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dependent;
import org.apache.dolphinscheduler.dao.repository.ProjectDao;
import org.apache.dolphinscheduler.dao.repository.TaskDefinitionDao;
@@ -23,10 +23,12 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.task.DependentLogicTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ILogicTaskPluginFactory;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
import lombok.extern.slf4j.Slf4j;
@@ -52,7 +54,8 @@ public class DependentLogicTaskPluginFactory implements ILogicTaskPluginFactory<
private IWorkflowRepository IWorkflowRepository;
@Override
- public DependentLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
+ public DependentLogicTask createLogicTask(final ITaskExecutor taskExecutor) throws LogicTaskInitializeException {
+ final TaskExecutionContext taskExecutionContext = taskExecutor.getTaskExecutionContext();
final int workflowInstanceId = taskExecutionContext.getWorkflowInstanceId();
final IWorkflowExecutionRunnable workflowExecutionRunnable = IWorkflowRepository.get(workflowInstanceId);
if (workflowExecutionRunnable == null) {
@@ -70,6 +73,6 @@ public class DependentLogicTaskPluginFactory implements ILogicTaskPluginFactory<
@Override
public String getTaskType() {
- return DependentLogicTask.TASK_TYPE;
+ return DependentLogicTaskChannelFactory.NAME;
}
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentTaskTracker.java
similarity index 89%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentTaskTracker.java
index bf8752a98d..f2cae5828e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dependent/DependentAsyncTaskExecuteFunction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dependent/DependentTaskTracker.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dependent;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dependent;
import static org.apache.dolphinscheduler.common.constants.Constants.DEPENDENT_SPLIT;
@@ -33,15 +33,14 @@ import org.apache.dolphinscheduler.dao.repository.WorkflowDefinitionDao;
import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DependResult;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentItem;
import org.apache.dolphinscheduler.plugin.task.api.model.DependentTaskModel;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DependentParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.DependentUtils;
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
import org.apache.dolphinscheduler.server.master.utils.DependentExecute;
-import java.time.Duration;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -56,9 +55,7 @@ import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction {
-
- private static final Duration DEFAULT_STATE_CHECK_INTERVAL = Duration.ofSeconds(10);
+public class DependentTaskTracker {
private final TaskExecutionContext taskExecutionContext;
private final DependentParameters dependentParameters;
@@ -73,13 +70,13 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
private final Map<String, DependResult> dependResultMap;
private final Map<String, Property> dependVarPoolPropertyMap;
- public DependentAsyncTaskExecuteFunction(TaskExecutionContext taskExecutionContext,
- DependentParameters dependentParameters,
- ProjectDao projectDao,
- WorkflowDefinitionDao workflowDefinitionDao,
- TaskDefinitionDao taskDefinitionDao,
- TaskInstanceDao taskInstanceDao,
- WorkflowInstanceDao workflowInstanceDao) {
+ public DependentTaskTracker(TaskExecutionContext taskExecutionContext,
+ DependentParameters dependentParameters,
+ ProjectDao projectDao,
+ WorkflowDefinitionDao workflowDefinitionDao,
+ TaskDefinitionDao taskDefinitionDao,
+ TaskInstanceDao taskInstanceDao,
+ WorkflowInstanceDao workflowInstanceDao) {
this.taskExecutionContext = taskExecutionContext;
this.dependentParameters = dependentParameters;
this.projectDao = projectDao;
@@ -95,8 +92,7 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
this.dependVarPoolPropertyMap = new HashMap<>();
}
- @Override
- public @NonNull AsyncTaskExecutionStatus getAsyncTaskExecutionStatus() {
+ public @NonNull TaskExecutionStatus getDependentTaskStatus() {
if (isAllDependentTaskFinished()) {
log.info("All dependent task finished, will calculate the dependent result");
DependResult dependResult = calculateDependResult();
@@ -104,12 +100,12 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
if (dependResult == DependResult.SUCCESS) {
dependentParameters.setVarPool(JSONUtils.toJsonString(dependVarPoolPropertyMap.values()));
log.info("Set dependentParameters varPool: {}", dependentParameters.getVarPool());
- return AsyncTaskExecutionStatus.SUCCESS;
+ return TaskExecutionStatus.SUCCESS;
} else {
- return AsyncTaskExecutionStatus.FAILED;
+ return TaskExecutionStatus.FAILURE;
}
}
- return AsyncTaskExecutionStatus.RUNNING;
+ return TaskExecutionStatus.RUNNING_EXECUTION;
}
private Date calculateDependentDate() {
@@ -239,9 +235,4 @@ public class DependentAsyncTaskExecuteFunction implements AsyncTaskExecuteFuncti
return isAllDependentTaskFinished;
}
- @Override
- public @NonNull Duration getAsyncTaskStateCheckInterval() {
- return dependentParameters.getDependence().getCheckInterval() == null ? DEFAULT_STATE_CHECK_INTERVAL
- : Duration.ofSeconds(dependentParameters.getDependence().getCheckInterval());
- }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicAsyncTaskExecuteFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicAsyncTaskExecuteFunction.java
similarity index 90%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicAsyncTaskExecuteFunction.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicAsyncTaskExecuteFunction.java
index 9f2b483579..b8ba05fe7a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicAsyncTaskExecuteFunction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicAsyncTaskExecuteFunction.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dynamic;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dynamic;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_DYNAMIC_START_PARAMS;
@@ -27,8 +27,8 @@ import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
import org.apache.dolphinscheduler.service.subworkflow.SubWorkflowService;
import java.time.Duration;
@@ -41,7 +41,7 @@ import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class DynamicAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction {
+public class DynamicAsyncTaskExecuteFunction {
private static final Duration TASK_EXECUTE_STATE_CHECK_INTERVAL = Duration.ofSeconds(10);
@@ -75,8 +75,7 @@ public class DynamicAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction
this.subWorkflowService = subWorkflowService;
}
- @Override
- public @NonNull AsyncTaskExecutionStatus getAsyncTaskExecutionStatus() {
+ public @NonNull TaskExecutionStatus getAsyncTaskExecutionStatus() {
List<WorkflowInstance> allSubWorkflowInstance = getAllSubProcessInstance();
int totalSubProcessInstanceCount = allSubWorkflowInstance.size();
@@ -90,16 +89,16 @@ public class DynamicAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction
if (successCount == totalSubProcessInstanceCount) {
log.info("all sub process instance success");
setOutputParameters();
- return AsyncTaskExecutionStatus.SUCCESS;
+ return TaskExecutionStatus.SUCCESS;
} else {
int failedCount = totalSubProcessInstanceCount - successCount;
log.info("failed sub process instance count: {}", failedCount);
- return AsyncTaskExecutionStatus.FAILED;
+ return TaskExecutionStatus.FAILURE;
}
}
if (logicTask.isCancel()) {
- return AsyncTaskExecutionStatus.FAILED;
+ return TaskExecutionStatus.FAILURE;
}
int runningCount = subWorkflowService.filterRunningProcessInstances(allSubWorkflowInstance).size();
@@ -109,7 +108,7 @@ public class DynamicAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction
startSubProcessInstances(allSubWorkflowInstance, startCount);
}
// query the status of sub workflow instance
- return AsyncTaskExecutionStatus.RUNNING;
+ return TaskExecutionStatus.RUNNING_EXECUTION;
}
private void setOutputParameters() {
@@ -143,7 +142,7 @@ public class DynamicAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction
List<Property> taskPropertyList = new ArrayList<>(JSONUtils.toList(taskInstance.getVarPool(), Property.class));
taskPropertyList.add(property);
- logicTask.getTaskParameters().setVarPool(JSONUtils.toJsonString(taskPropertyList));
+ // logicTask.getTaskParameters().setVarPool(JSONUtils.toJsonString(taskPropertyList));
log.info("set property: {}", property);
}
@@ -171,9 +170,4 @@ public class DynamicAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction
return subWorkflowService.getAllDynamicSubWorkflow(workflowInstance.getId(), taskInstance.getTaskCode());
}
- @Override
- public @NonNull Duration getAsyncTaskStateCheckInterval() {
- return TASK_EXECUTE_STATE_CHECK_INTERVAL;
- }
-
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicCommandUtils.java
similarity index 97%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicCommandUtils.java
index 567cd7ff88..2dc76c9e72 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicCommandUtils.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicCommandUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dynamic;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dynamic;
import org.apache.dolphinscheduler.common.constants.CommandKeyConstants;
import org.apache.dolphinscheduler.common.enums.CommandType;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicLogicTask.java
similarity index 86%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicLogicTask.java
index 6942260d8b..40fd7ab69b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicLogicTask.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dynamic;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dynamic;
import org.apache.dolphinscheduler.common.constants.CommandKeyConstants;
import org.apache.dolphinscheduler.common.enums.Flag;
@@ -35,13 +35,14 @@ import org.apache.dolphinscheduler.extract.master.IWorkflowControlClient;
import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowInstanceStopRequest;
import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowInstanceStopResponse;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.DynamicInputParameter;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.DynamicParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.AbstractLogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ITaskParameterDeserializer;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
-import org.apache.dolphinscheduler.server.master.runner.task.BaseAsyncLogicTask;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.subworkflow.SubWorkflowService;
@@ -61,7 +62,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.Lists;
@Slf4j
-public class DynamicLogicTask extends BaseAsyncLogicTask<DynamicParameters> {
+public class DynamicLogicTask extends AbstractLogicTask<DynamicParameters> {
public static final String TASK_TYPE = "DYNAMIC";
private final WorkflowInstanceDao workflowInstanceDao;
@@ -78,6 +79,8 @@ public class DynamicLogicTask extends BaseAsyncLogicTask<DynamicParameters> {
private TaskInstance taskInstance;
+ private final TaskExecutionContext taskExecutionContext;
+
private boolean haveBeenCanceled = false;
public DynamicLogicTask(TaskExecutionContext taskExecutionContext,
@@ -87,9 +90,8 @@ public class DynamicLogicTask extends BaseAsyncLogicTask<DynamicParameters> {
ProcessService processService,
WorkflowDefinitionMapper workflowDefinitionMapper,
CommandMapper commandMapper) {
- super(taskExecutionContext,
- JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<DynamicParameters>() {
- }));
+ super(taskExecutionContext);
+ this.taskExecutionContext = taskExecutionContext;
this.workflowInstanceDao = workflowInstanceDao;
this.subWorkflowService = subWorkflowService;
this.processService = processService;
@@ -100,28 +102,27 @@ public class DynamicLogicTask extends BaseAsyncLogicTask<DynamicParameters> {
this.taskInstance = taskInstanceDao.queryById(taskExecutionContext.getTaskInstanceId());
}
- @Override
- public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() throws MasterTaskExecuteException {
- List<Map<String, String>> parameterGroup = generateParameterGroup();
-
- if (parameterGroup.size() > taskParameters.getMaxNumOfSubWorkflowInstances()) {
- log.warn("the number of sub process instances [{}] exceeds the maximum limit [{}]", parameterGroup.size(),
- taskParameters.getMaxNumOfSubWorkflowInstances());
- parameterGroup = parameterGroup.subList(0, taskParameters.getMaxNumOfSubWorkflowInstances());
- }
-
- // if already exists sub process instance, do not generate again
- List<WorkflowInstance> existsSubWorkflowInstanceList =
- subWorkflowService.getAllDynamicSubWorkflow(workflowInstance.getId(), taskInstance.getTaskCode());
- if (CollectionUtils.isEmpty(existsSubWorkflowInstanceList)) {
- generateSubWorkflowInstance(parameterGroup);
- } else {
- resetProcessInstanceStatus(existsSubWorkflowInstanceList);
- }
- return new DynamicAsyncTaskExecuteFunction(taskExecutionContext, workflowInstance, taskInstance, this,
- commandMapper,
- subWorkflowService, taskParameters.getDegreeOfParallelism());
- }
+ // public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() throws MasterTaskExecuteException {
+ // List<Map<String, String>> parameterGroup = generateParameterGroup();
+ //
+ // if (parameterGroup.size() > dynamicParameters.getMaxNumOfSubWorkflowInstances()) {
+ // log.warn("the number of sub process instances [{}] exceeds the maximum limit [{}]", parameterGroup.size(),
+ // dynamicParameters.getMaxNumOfSubWorkflowInstances());
+ // parameterGroup = parameterGroup.subList(0, dynamicParameters.getMaxNumOfSubWorkflowInstances());
+ // }
+ //
+ // // if already exists sub process instance, do not generate again
+ // List<WorkflowInstance> existsSubWorkflowInstanceList =
+ // subWorkflowService.getAllDynamicSubWorkflow(workflowInstance.getId(), taskInstance.getTaskCode());
+ // if (CollectionUtils.isEmpty(existsSubWorkflowInstanceList)) {
+ // generateSubWorkflowInstance(parameterGroup);
+ // } else {
+ // resetProcessInstanceStatus(existsSubWorkflowInstanceList);
+ // }
+ // return new DynamicAsyncTaskExecuteFunction(taskExecutionContext, workflowInstance, taskInstance, this,
+ // commandMapper,
+ // subWorkflowService, dynamicParameters.getDegreeOfParallelism());
+ // }
public void resetProcessInstanceStatus(List<WorkflowInstance> existsSubWorkflowInstanceList) {
switch (workflowInstance.getCommandType()) {
@@ -250,6 +251,16 @@ public class DynamicLogicTask extends BaseAsyncLogicTask<DynamicParameters> {
return dynamicInputParameters;
}
+ @Override
+ public void start() throws MasterTaskExecuteException {
+ // todo:
+ }
+
+ @Override
+ public TaskExecutionStatus getTaskExecutionState() {
+ return taskExecutionContext.getCurrentExecutionStatus();
+ }
+
@Override
public void pause() throws MasterTaskExecuteException {
// todo: support pause
@@ -264,6 +275,12 @@ public class DynamicLogicTask extends BaseAsyncLogicTask<DynamicParameters> {
}
}
+ @Override
+ public ITaskParameterDeserializer<DynamicParameters> getTaskParameterDeserializer() {
+ return taskParamsJson -> JSONUtils.parseObject(taskParamsJson, new TypeReference<DynamicParameters>() {
+ });
+ }
+
private void doKillSubWorkflowInstances() throws MasterTaskExecuteException {
List<WorkflowInstance> existsSubWorkflowInstanceList =
subWorkflowService.getAllDynamicSubWorkflow(workflowInstance.getId(), taskInstance.getTaskCode());
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicLogicTaskPluginFactory.java
similarity index 76%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskPluginFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicLogicTaskPluginFactory.java
index cf9c599957..5065e80c7f 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicLogicTaskPluginFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicLogicTaskPluginFactory.java
@@ -15,16 +15,17 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dynamic;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dynamic;
import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkflowDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ILogicTaskPluginFactory;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.subworkflow.SubWorkflowService;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
import lombok.extern.slf4j.Slf4j;
@@ -54,10 +55,15 @@ public class DynamicLogicTaskPluginFactory implements ILogicTaskPluginFactory<Dy
SubWorkflowService subWorkflowService;
@Override
- public DynamicLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) {
- return new DynamicLogicTask(taskExecutionContext, workflowInstanceDao, taskInstanceDao, subWorkflowService,
+ public DynamicLogicTask createLogicTask(final ITaskExecutor taskExecutor) {
+ final TaskExecutionContext taskExecutionContext = taskExecutor.getTaskExecutionContext();
+ return new DynamicLogicTask(taskExecutionContext,
+ workflowInstanceDao,
+ taskInstanceDao,
+ subWorkflowService,
processService,
- processDefineMapper, commandMapper);
+ processDefineMapper,
+ commandMapper);
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicOutput.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicOutput.java
similarity index 92%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicOutput.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicOutput.java
index 666a59cafa..c727fcb08e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/dynamic/DynamicOutput.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/dynamic/DynamicOutput.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.dynamic;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.dynamic;
import java.util.Map;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/fake/LogicFakeTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTask.java
similarity index 56%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/fake/LogicFakeTask.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTask.java
index 8c95471f03..b05dc8cff4 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/fake/LogicFakeTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTask.java
@@ -15,19 +15,20 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.fake;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.fake;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.parameters.LogicFakeTaskParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
-import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.AbstractLogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ITaskParameterDeserializer;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
import lombok.extern.slf4j.Slf4j;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.annotations.VisibleForTesting;
/**
@@ -36,49 +37,65 @@ import com.google.common.annotations.VisibleForTesting;
*/
@Slf4j
@VisibleForTesting
-public class LogicFakeTask extends BaseSyncLogicTask<LogicFakeTaskParameters> {
-
- private volatile boolean killFlag;
+public class LogicFakeTask extends AbstractLogicTask<LogicFakeTaskParameters> {
private Process process;
- public LogicFakeTask(final IWorkflowExecutionRunnable workflowExecutionRunnable,
- final TaskExecutionContext taskExecutionContext) {
- super(workflowExecutionRunnable, taskExecutionContext,
- JSONUtils.parseObject(taskExecutionContext.getTaskParams(), LogicFakeTaskParameters.class));
+ public LogicFakeTask(final TaskExecutionContext taskExecutionContext) {
+ super(taskExecutionContext);
+ onTaskRunning();
}
@Override
- public void handle() throws MasterTaskExecuteException {
+ public void start() throws MasterTaskExecuteException {
try {
+ log.info("Begin to execute LogicFakeTask: {}", taskExecutionContext.getTaskName());
+
final String shellScript = ParameterUtils.convertParameterPlaceholders(
taskParameters.getShellScript(),
ParameterUtils.convert(taskExecutionContext.getPrepareParamsMap()));
final String[] cmd = {"/bin/sh", "-c", shellScript};
process = Runtime.getRuntime().exec(cmd);
int exitCode = process.waitFor();
- if (killFlag) {
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.KILL);
+ if (taskExecutionStatus != TaskExecutionStatus.RUNNING_EXECUTION) {
+ // The task has been killed
return;
}
if (exitCode == 0) {
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
+ log.info("LogicFakeTask: {} execute success with exit code: {}",
+ taskExecutionContext.getTaskName(),
+ exitCode);
+ onTaskSuccess();
} else {
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.FAILURE);
+ log.info("LogicFakeTask: {} execute failed with exit code: {}",
+ taskExecutionContext.getTaskName(),
+ exitCode);
+ onTaskFailed();
}
} catch (Exception ex) {
throw new MasterTaskExecuteException("FakeTask execute failed", ex);
}
}
+ @Override
+ public void pause() throws MasterTaskExecuteException {
+ log.info("LogicFakeTask: {} doesn't support pause", taskExecutionContext.getTaskName());
+ }
+
@Override
public void kill() throws MasterTaskExecuteException {
- log.info("kill task : {}", taskExecutionContext.getTaskName());
if (process != null && process.isAlive()) {
- killFlag = true;
- process.destroy();
- log.info("kill task : {} succeed", taskExecutionContext.getTaskName());
+ // todo: use shell script to kill the process?
+ onTaskKilled();
+ process.destroyForcibly();
+ log.info("Kill LogicFakeTask: {} succeed", taskExecutionContext.getTaskName());
}
}
+ @Override
+ public ITaskParameterDeserializer<LogicFakeTaskParameters> getTaskParameterDeserializer() {
+ return taskParamsJson -> JSONUtils.parseObject(taskParamsJson, new TypeReference<LogicFakeTaskParameters>() {
+ });
+ }
+
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/fake/LogicFakeTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTaskPluginFactory.java
similarity index 56%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/fake/LogicFakeTaskPluginFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTaskPluginFactory.java
index 8fc2ea9633..8ebcdbe1f0 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/fake/LogicFakeTaskPluginFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/fake/LogicFakeTaskPluginFactory.java
@@ -15,15 +15,13 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.fake;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.fake;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.task.LogicFakeTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
-import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ILogicTaskPluginFactory;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.google.common.annotations.VisibleForTesting;
@@ -32,18 +30,15 @@ import com.google.common.annotations.VisibleForTesting;
@VisibleForTesting
public class LogicFakeTaskPluginFactory implements ILogicTaskPluginFactory<LogicFakeTask> {
- @Autowired
- private IWorkflowRepository IWorkflowRepository;
+ private final IWorkflowRepository workflowRepository;
+
+ public LogicFakeTaskPluginFactory(final IWorkflowRepository workflowRepository) {
+ this.workflowRepository = workflowRepository;
+ }
@Override
- public LogicFakeTask createLogicTask(final TaskExecutionContext taskExecutionContext) {
- final IWorkflowExecutionRunnable workflowExecutionRunnable =
- IWorkflowRepository.get(taskExecutionContext.getWorkflowInstanceId());
- if (workflowExecutionRunnable == null) {
- throw new IllegalStateException(
- "Cannot find the WorkflowExecuteRunnable: " + taskExecutionContext.getWorkflowInstanceId());
- }
- return new LogicFakeTask(workflowExecutionRunnable, taskExecutionContext);
+ public LogicFakeTask createLogicTask(final ITaskExecutor taskExecutor) {
+ return new LogicFakeTask(taskExecutor.getTaskExecutionContext());
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowControlClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowControlClient.java
similarity index 96%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowControlClient.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowControlClient.java
index 69109f09dc..e26fa5775b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowControlClient.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowControlClient.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.subworkflow;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.subworkflow;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
@@ -31,10 +31,10 @@ import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowI
import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowInstanceStopRequest;
import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowInstanceStopResponse;
import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowManualTriggerRequest;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.subworkflow.trigger.SubWorkflowManualTrigger;
import org.apache.dolphinscheduler.server.master.engine.workflow.trigger.WorkflowInstanceRecoverFailureTaskTrigger;
import org.apache.dolphinscheduler.server.master.engine.workflow.trigger.WorkflowInstanceRecoverSuspendTaskTrigger;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.task.subworkflow.trigger.SubWorkflowManualTrigger;
import lombok.extern.slf4j.Slf4j;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTask.java
similarity index 87%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTask.java
index d5f21bfc53..89f8b1ecb7 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTask.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.subworkflow;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.subworkflow;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
@@ -34,12 +34,14 @@ import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowI
import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowInstanceStopResponse;
import org.apache.dolphinscheduler.extract.master.transportor.workflow.WorkflowManualTriggerRequest;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SubWorkflowParameters;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.AbstractLogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ITaskParameterDeserializer;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-import org.apache.dolphinscheduler.server.master.runner.task.BaseAsyncLogicTask;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorRuntimeContextChangedLifecycleEvent;
import lombok.extern.slf4j.Slf4j;
@@ -48,7 +50,7 @@ import org.springframework.context.ApplicationContext;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j
-public class SubWorkflowLogicTask extends BaseAsyncLogicTask<SubWorkflowParameters> {
+public class SubWorkflowLogicTask extends AbstractLogicTask<SubWorkflowParameters> {
private SubWorkflowLogicTaskRuntimeContext subWorkflowLogicTaskRuntimeContext;
@@ -56,35 +58,48 @@ public class SubWorkflowLogicTask extends BaseAsyncLogicTask<SubWorkflowParamete
private final ApplicationContext applicationContext;
+ private SubWorkflowTracker subWorkflowTracker;
+
+ private ITaskExecutor taskExecutor;
+
public SubWorkflowLogicTask(final TaskExecutionContext taskExecutionContext,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
+ final ITaskExecutor taskExecutor,
final ApplicationContext applicationContext) {
- super(taskExecutionContext,
- JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<SubWorkflowParameters>() {
- }));
+ super(taskExecutionContext);
+ this.taskExecutor = taskExecutor;
this.workflowExecutionRunnable = workflowExecutionRunnable;
this.applicationContext = applicationContext;
this.subWorkflowLogicTaskRuntimeContext = JSONUtils.parseObject(
taskExecutionContext.getAppIds(),
SubWorkflowLogicTaskRuntimeContext.class);
+ onTaskRunning();
}
@Override
- public AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() {
+ public void start() throws MasterTaskExecuteException {
subWorkflowLogicTaskRuntimeContext = initializeSubWorkflowInstance();
upsertSubWorkflowRelation();
taskExecutionContext.setAppIds(JSONUtils.toJsonString(subWorkflowLogicTaskRuntimeContext));
- applicationContext
- .getBean(LogicTaskInstanceExecutionEventSenderManager.class)
- .runningEventSender()
- .sendMessage(taskExecutionContext);
+ taskExecutor.getTaskExecutorEventBus()
+ .publish(TaskExecutorRuntimeContextChangedLifecycleEvent.of(taskExecutor));
- return new SubWorkflowAsyncTaskExecuteFunction(
+ subWorkflowTracker = new SubWorkflowTracker(
subWorkflowLogicTaskRuntimeContext,
applicationContext.getBean(WorkflowInstanceDao.class));
}
+ @Override
+ public TaskExecutionStatus getTaskExecutionState() {
+ if (subWorkflowTracker == null) {
+ // The sub workflow has not been started
+ return taskExecutionStatus;
+ }
+ taskExecutionStatus = subWorkflowTracker.getSubWorkflowState();
+ return taskExecutionStatus;
+ }
+
@Override
public void pause() throws MasterTaskExecuteException {
if (subWorkflowLogicTaskRuntimeContext == null) {
@@ -120,6 +135,12 @@ public class SubWorkflowLogicTask extends BaseAsyncLogicTask<SubWorkflowParamete
}
}
+ @Override
+ public ITaskParameterDeserializer<SubWorkflowParameters> getTaskParameterDeserializer() {
+ return taskParamsJson -> JSONUtils.parseObject(taskParamsJson, new TypeReference<SubWorkflowParameters>() {
+ });
+ }
+
private SubWorkflowLogicTaskRuntimeContext initializeSubWorkflowInstance() {
// todo: doFailover if the runtime context is not null and task is generated by failover
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTaskPluginFactory.java
similarity index 61%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskPluginFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTaskPluginFactory.java
index a11d3d0a74..f3f9ebc9e1 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskPluginFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTaskPluginFactory.java
@@ -15,18 +15,17 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.subworkflow;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.subworkflow;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ILogicTaskPluginFactory;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
@@ -34,21 +33,26 @@ import org.springframework.stereotype.Component;
@Component
public class SubWorkflowLogicTaskPluginFactory implements ILogicTaskPluginFactory<SubWorkflowLogicTask> {
- @Autowired
- private ApplicationContext applicationContext;
+ private final ApplicationContext applicationContext;
- @Autowired
- private IWorkflowRepository IWorkflowRepository;
+ private final IWorkflowRepository workflowRepository;
+
+ public SubWorkflowLogicTaskPluginFactory(final ApplicationContext applicationContext,
+ final IWorkflowRepository workflowRepository) {
+ this.applicationContext = applicationContext;
+ this.workflowRepository = workflowRepository;
+ }
@Override
- public SubWorkflowLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
+ public SubWorkflowLogicTask createLogicTask(final ITaskExecutor taskExecutor) {
+ final TaskExecutionContext taskExecutionContext = taskExecutor.getTaskExecutionContext();
final int workflowInstanceId = taskExecutionContext.getWorkflowInstanceId();
- final IWorkflowExecutionRunnable workflowExecutionRunnable = IWorkflowRepository.get(workflowInstanceId);
- if (workflowExecutionRunnable == null) {
- throw new LogicTaskInitializeException(
- "Cannot find the WorkflowExecuteRunnable by : " + workflowInstanceId);
- }
- return new SubWorkflowLogicTask(taskExecutionContext, workflowExecutionRunnable, applicationContext);
+ final IWorkflowExecutionRunnable workflowExecutionRunnable = workflowRepository.get(workflowInstanceId);
+ return new SubWorkflowLogicTask(
+ taskExecutionContext,
+ workflowExecutionRunnable,
+ taskExecutor,
+ applicationContext);
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskRuntimeContext.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTaskRuntimeContext.java
similarity index 94%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskRuntimeContext.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTaskRuntimeContext.java
index 48d0815735..20b170cfb1 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowLogicTaskRuntimeContext.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowLogicTaskRuntimeContext.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.subworkflow;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.subworkflow;
import lombok.AllArgsConstructor;
import lombok.Builder;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowAsyncTaskExecuteFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowTracker.java
similarity index 61%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowAsyncTaskExecuteFunction.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowTracker.java
index 8938469762..4cc1655608 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/SubWorkflowAsyncTaskExecuteFunction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/SubWorkflowTracker.java
@@ -15,56 +15,47 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.subworkflow;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.subworkflow;
import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
import org.apache.dolphinscheduler.dao.repository.WorkflowInstanceDao;
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
-
-import java.time.Duration;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class SubWorkflowAsyncTaskExecuteFunction implements AsyncTaskExecuteFunction {
-
- private static final Duration SUB_WORKFLOW_TASK_EXECUTE_STATE_CHECK_INTERVAL = Duration.ofSeconds(10);
+public class SubWorkflowTracker {
private final WorkflowInstanceDao workflowInstanceDao;
private final SubWorkflowLogicTaskRuntimeContext subWorkflowLogicTaskRuntimeContext;
- public SubWorkflowAsyncTaskExecuteFunction(final SubWorkflowLogicTaskRuntimeContext subWorkflowLogicTaskRuntimeContext,
- final WorkflowInstanceDao workflowInstanceDao) {
+ public SubWorkflowTracker(final SubWorkflowLogicTaskRuntimeContext subWorkflowLogicTaskRuntimeContext,
+ final WorkflowInstanceDao workflowInstanceDao) {
this.subWorkflowLogicTaskRuntimeContext = subWorkflowLogicTaskRuntimeContext;
this.workflowInstanceDao = workflowInstanceDao;
}
- @Override
- public @NonNull AsyncTaskExecutionStatus getAsyncTaskExecutionStatus() {
+ public @NonNull TaskExecutionStatus getSubWorkflowState() {
final Integer subWorkflowInstanceId = subWorkflowLogicTaskRuntimeContext.getSubWorkflowInstanceId();
final WorkflowInstance subWorkflowInstance = workflowInstanceDao.queryById(subWorkflowInstanceId);
if (subWorkflowInstance == null) {
log.info("Cannot find the SubWorkflow instance: {}, maybe it has been deleted", subWorkflowInstanceId);
- return AsyncTaskExecutionStatus.FAILED;
+ return TaskExecutionStatus.FAILURE;
}
switch (subWorkflowInstance.getState()) {
case PAUSE:
- return AsyncTaskExecutionStatus.PAUSE;
+ return TaskExecutionStatus.PAUSE;
case STOP:
- return AsyncTaskExecutionStatus.KILL;
+ return TaskExecutionStatus.KILL;
case SUCCESS:
- return AsyncTaskExecutionStatus.SUCCESS;
+ return TaskExecutionStatus.SUCCESS;
case FAILURE:
- return AsyncTaskExecutionStatus.FAILED;
+ return TaskExecutionStatus.FAILURE;
default:
- return AsyncTaskExecutionStatus.RUNNING;
+ return TaskExecutionStatus.RUNNING_EXECUTION;
}
}
- @Override
- public @NonNull Duration getAsyncTaskStateCheckInterval() {
- return SUB_WORKFLOW_TASK_EXECUTE_STATE_CHECK_INTERVAL;
- }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/trigger/SubWorkflowManualTrigger.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/trigger/SubWorkflowManualTrigger.java
similarity index 94%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/trigger/SubWorkflowManualTrigger.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/trigger/SubWorkflowManualTrigger.java
index 5484de2128..cbbc903008 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/subworkflow/trigger/SubWorkflowManualTrigger.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/subworkflow/trigger/SubWorkflowManualTrigger.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.subworkflow.trigger;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.subworkflow.trigger;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/switchtask/SwitchLogicTask.java
similarity index 86%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/switchtask/SwitchLogicTask.java
index 2407a7f0ab..60c78d7d5a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTask.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/switchtask/SwitchLogicTask.java
@@ -15,20 +15,19 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.switchtask;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.switchtask;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.AbstractLogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ITaskParameterDeserializer;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
import org.apache.dolphinscheduler.server.master.runner.IWorkflowExecuteContext;
-import org.apache.dolphinscheduler.server.master.runner.task.BaseSyncLogicTask;
import org.apache.dolphinscheduler.server.master.utils.SwitchTaskUtils;
import org.apache.commons.collections4.CollectionUtils;
@@ -43,29 +42,25 @@ import lombok.extern.slf4j.Slf4j;
import com.fasterxml.jackson.core.type.TypeReference;
@Slf4j
-public class SwitchLogicTask extends BaseSyncLogicTask<SwitchParameters> {
-
- public static final String TASK_TYPE = "SWITCH";
+public class SwitchLogicTask extends AbstractLogicTask<SwitchParameters> {
private final IWorkflowExecutionRunnable workflowExecutionRunnable;
private final TaskInstance taskInstance;
public SwitchLogicTask(IWorkflowExecutionRunnable workflowExecutionRunnable,
TaskExecutionContext taskExecutionContext) {
- super(workflowExecutionRunnable,
- taskExecutionContext,
- JSONUtils.parseObject(taskExecutionContext.getTaskParams(), new TypeReference<SwitchParameters>() {
- }));
+ super(taskExecutionContext);
this.workflowExecutionRunnable = workflowExecutionRunnable;
this.taskInstance = workflowExecutionRunnable
.getWorkflowExecuteContext()
.getWorkflowExecutionGraph()
.getTaskExecutionRunnableById(taskExecutionContext.getTaskInstanceId())
.getTaskInstance();
+ onTaskRunning();
}
@Override
- public void handle() throws MasterTaskExecuteException {
+ public void start() {
if (CollectionUtils.isEmpty(taskParameters.getSwitchResult().getDependTaskList())) {
// If the branch is empty then will go into the default branch
// This case shouldn't happen, we can directly throw exception and forbid the user to set branch
@@ -76,8 +71,9 @@ public class SwitchLogicTask extends BaseSyncLogicTask<SwitchParameters> {
}
checkIfBranchExist(taskParameters.getNextBranch());
taskInstance.setTaskParams(JSONUtils.toJsonString(taskParameters));
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
- log.info("Switch task execute finished: {}", taskExecutionContext.getCurrentExecutionStatus().name());
+
+ onTaskSuccess();
+ log.info("Switch task execute finished");
}
private void moveToDefaultBranch() {
@@ -144,4 +140,20 @@ public class SwitchLogicTask extends BaseSyncLogicTask<SwitchParameters> {
.orElse(null);
}
+ @Override
+ public void pause() {
+ log.info("The SwitchTask does not support pause operation");
+ }
+
+ @Override
+ public void kill() {
+ log.info("The SwitchTask does not support kill operation");
+ }
+
+ @Override
+ public ITaskParameterDeserializer<SwitchParameters> getTaskParameterDeserializer() {
+ return taskParamsJson -> JSONUtils.parseObject(taskParamsJson, new TypeReference<SwitchParameters>() {
+ });
+ }
+
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/switchtask/SwitchLogicTaskPluginFactory.java
similarity index 77%
rename from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java
rename to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/switchtask/SwitchLogicTaskPluginFactory.java
index ce5fb6dfe6..9bb0661635 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/switchtask/SwitchLogicTaskPluginFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/executor/plugin/switchtask/SwitchLogicTaskPluginFactory.java
@@ -15,13 +15,15 @@
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.master.runner.task.switchtask;
+package org.apache.dolphinscheduler.server.master.engine.executor.plugin.switchtask;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.plugin.task.api.task.SwitchLogicTaskChannelFactory;
import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.ILogicTaskPluginFactory;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTaskPluginFactory;
+import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
import lombok.extern.slf4j.Slf4j;
@@ -36,7 +38,8 @@ public class SwitchLogicTaskPluginFactory implements ILogicTaskPluginFactory<Swi
private IWorkflowRepository IWorkflowRepository;
@Override
- public SwitchLogicTask createLogicTask(TaskExecutionContext taskExecutionContext) throws LogicTaskInitializeException {
+ public SwitchLogicTask createLogicTask(final ITaskExecutor taskExecutor) throws LogicTaskInitializeException {
+ final TaskExecutionContext taskExecutionContext = taskExecutor.getTaskExecutionContext();
final int workflowInstanceId = taskExecutionContext.getWorkflowInstanceId();
IWorkflowExecutionRunnable workflowExecutionRunnable =
IWorkflowRepository.get(workflowInstanceId);
@@ -49,6 +52,6 @@ public class SwitchLogicTaskPluginFactory implements ILogicTaskPluginFactory<Swi
@Override
public String getTaskType() {
- return SwitchLogicTask.TASK_TYPE;
+ return SwitchLogicTaskChannelFactory.NAME;
}
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/SuccessorFlowAdjuster.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/SuccessorFlowAdjuster.java
index a9d526f254..01594d105f 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/SuccessorFlowAdjuster.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/graph/SuccessorFlowAdjuster.java
@@ -22,9 +22,9 @@ import org.apache.dolphinscheduler.plugin.task.api.model.SwitchResultVo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.ConditionsParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SwitchParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.condition.ConditionLogicTask;
+import org.apache.dolphinscheduler.server.master.engine.executor.plugin.switchtask.SwitchLogicTask;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.runner.task.condition.ConditionLogicTask;
-import org.apache.dolphinscheduler.server.master.runner.task.switchtask.SwitchLogicTask;
import org.apache.commons.collections4.CollectionUtils;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/SystemEventBus.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/SystemEventBus.java
index c57014c4b8..bdba080b79 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/SystemEventBus.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/SystemEventBus.java
@@ -23,8 +23,6 @@ import org.apache.dolphinscheduler.server.master.engine.system.event.AbstractSys
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.AbstractTaskLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.workflow.lifecycle.AbstractWorkflowLifecycleLifecycleEvent;
-import java.util.concurrent.LinkedBlockingQueue;
-
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@@ -37,24 +35,12 @@ import org.springframework.stereotype.Component;
@Component
public class SystemEventBus extends AbstractDelayEventBus<AbstractSystemEvent> {
- private final LinkedBlockingQueue<AbstractSystemEvent> eventChannel = new LinkedBlockingQueue<>();
-
public void publish(final AbstractSystemEvent event) {
- try {
- eventChannel.put(event);
- log.info("Published SystemEvent: {}", event);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IllegalStateException("The thread has been interrupted", e);
- }
+ super.publish(event);
+ log.info("Published SystemEvent: {}", event);
}
public AbstractSystemEvent take() throws InterruptedException {
- return eventChannel.take();
+ return delayEventQueue.take();
}
-
- public boolean isEmpty() {
- return eventChannel.isEmpty();
- }
-
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/AbstractSystemEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/AbstractSystemEvent.java
index 17c1dfb296..7680c9cf55 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/AbstractSystemEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/AbstractSystemEvent.java
@@ -23,6 +23,10 @@ import java.util.Date;
public abstract class AbstractSystemEvent extends AbstractDelayEvent {
+ public AbstractSystemEvent() {
+ super();
+ }
+
public AbstractSystemEvent(long delayTime) {
super(delayTime);
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/GlobalMasterFailoverEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/GlobalMasterFailoverEvent.java
index a6c7909851..187d56fd47 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/GlobalMasterFailoverEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/system/event/GlobalMasterFailoverEvent.java
@@ -29,7 +29,7 @@ public class GlobalMasterFailoverEvent extends AbstractSystemEvent {
private final Date eventTime;
public GlobalMasterFailoverEvent(Date eventTime) {
- super(eventTime.getTime());
+ super();
this.eventTime = eventTime;
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClient.java
index 55ce24972c..8c6c02ef76 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClient.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClient.java
@@ -19,13 +19,31 @@ package org.apache.dolphinscheduler.server.master.engine.task.client;
import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskKillException;
import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskPauseException;
+import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskReassignMasterHostException;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
+import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
+import org.apache.dolphinscheduler.task.executor.TaskEngine;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
/**
- * The client of task executor, used to communicate with task executor.
+ * The client used to communicate with {@link TaskEngine}.
*/
public interface ITaskExecutorClient {
+ /**
+ * Dispatch the task to task executor.
+ *
+ * @throws TaskDispatchException If dispatch failed or error occurs.
+ */
+ void dispatch(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskDispatchException;
+
+ /**
+ * Reassign the workflow instance host from task executor.
+ *
+ * @throws TaskReassignMasterHostException If an error occurs.
+ */
+ boolean reassignWorkflowInstanceHost(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskReassignMasterHostException;
+
/**
* Pause task from task executor.
* <p> This method is not a sync method, it will return immediately after send a kill request to remote executor and receive a response.
@@ -34,7 +52,7 @@ public interface ITaskExecutorClient {
*
* @throws TaskPauseException If an error occurs.
*/
- void pause(ITaskExecutionRunnable taskExecutionRunnable) throws TaskPauseException;
+ void pause(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskPauseException;
/**
* Kill task from task executor.
@@ -44,5 +62,14 @@ public interface ITaskExecutorClient {
*
* @throws TaskKillException If an error occurs.
*/
- void kill(ITaskExecutionRunnable taskExecutionRunnable) throws TaskKillException;
+ void kill(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskKillException;
+
+ /**
+ * Send TaskExecutorLifecycleEventAck to TaskEngine.
+ * <p> This method will not throw exception, once send ack failed, the executor engine will retry.
+ */
+ void ackTaskExecutorLifecycleEvent(
+ final ITaskExecutionRunnable taskExecutionRunnable,
+ final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck);
+
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClientDelegator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClientDelegator.java
index d157ea8f58..659fde2cfa 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClientDelegator.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskExecutorClientDelegator.java
@@ -18,6 +18,8 @@
package org.apache.dolphinscheduler.server.master.engine.task.client;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
+import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
/**
* The interface of task executor client delegator. It is used to send operation to task executor server.
@@ -27,9 +29,32 @@ import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecu
*/
public interface ITaskExecutorClientDelegator {
- void dispatch(final ITaskExecutionRunnable taskExecutionRunnable);
+ /**
+ * Dispatch the task to task executor.
+ *
+ * @throws TaskDispatchException If dispatch failed
+ */
+ void dispatch(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskDispatchException;
+ /**
+ * Take over the task from task executor.
+ */
+ boolean reassignMasterHost(final ITaskExecutionRunnable taskExecutionRunnable);
+
+ /**
+ * Pause the task, this method doesn't guarantee the task is paused success.
+ */
void pause(final ITaskExecutionRunnable taskExecutionRunnable);
+ /**
+ * Kill the task, this method doesn't guarantee the task is killed success.
+ */
void kill(final ITaskExecutionRunnable taskExecutionRunnable);
+
+ /**
+ * Ack the task executor lifecycle event.
+ */
+ void ackTaskExecutorLifecycleEvent(
+ final ITaskExecutionRunnable taskExecutionRunnable,
+ final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck);
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskOperator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskOperator.java
deleted file mode 100644
index 9b01b52cca..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/ITaskOperator.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.engine.task.client;
-
-import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskKillException;
-import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-
-public interface ITaskOperator {
-
- void dispatch(ITaskExecutionRunnable taskExecutionRunnable);
-
- void pause(ITaskExecutionRunnable taskExecutionRunnable);
-
- void kill(ITaskExecutionRunnable taskExecutionRunnable) throws TaskKillException;
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/LogicTaskExecutorClientDelegator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/LogicTaskExecutorClientDelegator.java
index 7a941f9e30..4de91b2a4b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/LogicTaskExecutorClientDelegator.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/LogicTaskExecutorClientDelegator.java
@@ -21,27 +21,57 @@ import static com.google.common.base.Preconditions.checkArgument;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.extract.base.client.Clients;
-import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse;
+import org.apache.dolphinscheduler.extract.master.ILogicTaskExecutorOperator;
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskKillException;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
+import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseResponse;
import org.apache.commons.lang3.StringUtils;
import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class LogicTaskExecutorClientDelegator implements ITaskExecutorClientDelegator {
+ @Autowired
+ private MasterConfig masterConfig;
+
@Override
- public void dispatch(final ITaskExecutionRunnable taskExecutionRunnable) {
+ public void dispatch(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskDispatchException {
+ final String logicTaskExecutorAddress = masterConfig.getMasterAddress();
+ final TaskExecutionContext taskExecutionContext = taskExecutionRunnable.getTaskExecutionContext();
+
+ taskExecutionContext.setHost(logicTaskExecutorAddress);
+ taskExecutionRunnable.getTaskInstance().setHost(logicTaskExecutorAddress);
+
+ final TaskExecutorDispatchResponse logicTaskDispatchResponse = Clients
+ .withService(ILogicTaskExecutorOperator.class)
+ .withHost(logicTaskExecutorAddress)
+ .dispatchTask(TaskExecutorDispatchRequest.of(taskExecutionContext));
+ if (!logicTaskDispatchResponse.isDispatchSuccess()) {
+ throw new TaskDispatchException(
+ String.format("Dispatch LogicTask to %s failed, response is: %s",
+ taskExecutionContext.getHost(), logicTaskDispatchResponse));
+ }
+ }
+ @Override
+ public boolean reassignMasterHost(final ITaskExecutionRunnable taskExecutionRunnable) {
+ // The Logic Task doesn't support take-over, since the logic task is not executed on the worker.
+ return false;
}
@Override
@@ -51,10 +81,10 @@ public class LogicTaskExecutorClientDelegator implements ITaskExecutorClientDele
final String taskName = taskInstance.getName();
checkArgument(StringUtils.isNotEmpty(executorHost), "Executor host is empty");
- final LogicTaskPauseResponse pauseResponse = Clients
- .withService(ILogicTaskInstanceOperator.class)
+ final TaskExecutorPauseResponse pauseResponse = Clients
+ .withService(ILogicTaskExecutorOperator.class)
.withHost(taskInstance.getHost())
- .pauseLogicTask(new LogicTaskPauseRequest(taskInstance.getId()));
+ .pauseTask(TaskExecutorPauseRequest.of(taskInstance.getId()));
if (pauseResponse.isSuccess()) {
log.info("Pause task {} on executor {} successfully", taskName, executorHost);
} else {
@@ -69,14 +99,29 @@ public class LogicTaskExecutorClientDelegator implements ITaskExecutorClientDele
final String taskName = taskInstance.getName();
checkArgument(StringUtils.isNotEmpty(executorHost), "Executor host is empty");
- final LogicTaskKillResponse killResponse = Clients
- .withService(ILogicTaskInstanceOperator.class)
+ final TaskExecutorKillResponse killResponse = Clients
+ .withService(ILogicTaskExecutorOperator.class)
.withHost(taskInstance.getHost())
- .killLogicTask(new LogicTaskKillRequest(taskInstance.getId()));
+ .killTask(TaskExecutorKillRequest.of(taskInstance.getId()));
if (killResponse.isSuccess()) {
log.info("Kill task {} on executor {} successfully", taskName, executorHost);
} else {
log.warn("Kill task {} on executor {} failed with response {}", taskName, executorHost, killResponse);
}
}
+
+ @Override
+ public void ackTaskExecutorLifecycleEvent(
+ final ITaskExecutionRunnable taskExecutionRunnable,
+ final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck) {
+ final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance();
+ final String executorHost = taskInstance.getHost();
+ checkArgument(StringUtils.isNotEmpty(executorHost), "Executor host is empty");
+
+ Clients
+ .withService(ILogicTaskExecutorOperator.class)
+ .withHost(taskInstance.getHost())
+ .ackTaskExecutorLifecycleEvent(taskExecutorLifecycleEventAck);
+ }
+
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/PhysicalTaskExecutorClientDelegator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/PhysicalTaskExecutorClientDelegator.java
index 2db50f84c9..f546033436 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/PhysicalTaskExecutorClientDelegator.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/PhysicalTaskExecutorClientDelegator.java
@@ -21,26 +21,108 @@ import static com.google.common.base.Preconditions.checkArgument;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.extract.base.client.Clients;
-import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseResponse;
+import org.apache.dolphinscheduler.extract.base.utils.Host;
+import org.apache.dolphinscheduler.extract.worker.IPhysicalTaskExecutorOperator;
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.master.cluster.loadbalancer.IWorkerLoadBalancer;
+import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
+import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorReassignMasterRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorReassignMasterResponse;
import org.apache.commons.lang3.StringUtils;
import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class PhysicalTaskExecutorClientDelegator implements ITaskExecutorClientDelegator {
+ @Autowired
+ private MasterConfig masterConfig;
+
+ @Autowired
+ private IWorkerLoadBalancer workerLoadBalancer;
+
@Override
- public void dispatch(final ITaskExecutionRunnable taskExecutionRunnable) {
+ public void dispatch(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskDispatchException {
+ final TaskExecutionContext taskExecutionContext = taskExecutionRunnable.getTaskExecutionContext();
+ final String taskName = taskExecutionContext.getTaskName();
+ final String physicalTaskExecutorAddress = workerLoadBalancer
+ .select(taskExecutionContext.getWorkerGroup())
+ .map(Host::of)
+ .map(Host::getAddress)
+ .orElseThrow(() -> new TaskDispatchException(
+ String.format("Cannot find the host to dispatch Task[id=%s, name=%s]",
+ taskExecutionContext.getTaskInstanceId(), taskName)));
+
+ taskExecutionContext.setHost(physicalTaskExecutorAddress);
+ taskExecutionRunnable.getTaskInstance().setHost(physicalTaskExecutorAddress);
+
+ try {
+ final TaskExecutorDispatchResponse taskExecutorDispatchResponse = Clients
+ .withService(IPhysicalTaskExecutorOperator.class)
+ .withHost(physicalTaskExecutorAddress)
+ .dispatchTask(TaskExecutorDispatchRequest.of(taskExecutionRunnable.getTaskExecutionContext()));
+ if (!taskExecutorDispatchResponse.isDispatchSuccess()) {
+ throw new TaskDispatchException(
+ "Dispatch task: " + taskName + " to " + physicalTaskExecutorAddress + " failed: "
+ + taskExecutorDispatchResponse);
+ }
+ } catch (TaskDispatchException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new TaskDispatchException(
+ "Dispatch task: " + taskName + " to " + physicalTaskExecutorAddress + " failed", e);
+ }
+ }
+
+ @Override
+ public boolean reassignMasterHost(final ITaskExecutionRunnable taskExecutionRunnable) {
+ final String taskName = taskExecutionRunnable.getName();
+ checkArgument(taskExecutionRunnable.isTaskInstanceInitialized(),
+ "Task " + taskName + "is not initialized cannot take-over");
+
+ final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance();
+ final String taskExecutorHost = taskInstance.getHost();
+ if (StringUtils.isEmpty(taskExecutorHost)) {
+ log.debug(
+ "The task executor: {} host is empty, cannot take-over, this might caused by the task hasn't dispatched",
+ taskName);
+ return false;
+ }
+ final TaskExecutorReassignMasterRequest taskExecutorReassignMasterRequest =
+ TaskExecutorReassignMasterRequest.builder()
+ .taskInstanceId(taskInstance.getId())
+ .workflowHost(masterConfig.getMasterAddress())
+ .build();
+ final TaskExecutorReassignMasterResponse taskExecutorReassignMasterResponse =
+ Clients
+ .withService(IPhysicalTaskExecutorOperator.class)
+ .withHost(taskInstance.getHost())
+ .reassignWorkflowInstanceHost(taskExecutorReassignMasterRequest);
+ boolean success = taskExecutorReassignMasterResponse.isSuccess();
+ if (success) {
+ log.info("Reassign master host {} to {} successfully", taskExecutorHost, taskName);
+ } else {
+ log.info("Reassign master host {} on {} failed with response {}",
+ taskExecutorHost,
+ taskName,
+ taskExecutorReassignMasterResponse);
+ }
+ return success;
}
@Override
@@ -50,10 +132,10 @@ public class PhysicalTaskExecutorClientDelegator implements ITaskExecutorClientD
final String taskName = taskInstance.getName();
checkArgument(StringUtils.isNotEmpty(executorHost), "Executor host is empty");
- final TaskInstancePauseResponse pauseResponse = Clients
- .withService(ITaskInstanceOperator.class)
+ final TaskExecutorPauseResponse pauseResponse = Clients
+ .withService(IPhysicalTaskExecutorOperator.class)
.withHost(taskInstance.getHost())
- .pauseTask(new TaskInstancePauseRequest(taskInstance.getId()));
+ .pauseTask(TaskExecutorPauseRequest.of(taskInstance.getId()));
if (pauseResponse.isSuccess()) {
log.info("Pause task {} on executor {} successfully", taskName, executorHost);
} else {
@@ -68,14 +150,28 @@ public class PhysicalTaskExecutorClientDelegator implements ITaskExecutorClientD
final String taskName = taskInstance.getName();
checkArgument(StringUtils.isNotEmpty(executorHost), "Executor host is empty");
- final TaskInstanceKillResponse killResponse = Clients
- .withService(ITaskInstanceOperator.class)
+ final TaskExecutorKillResponse killResponse = Clients
+ .withService(IPhysicalTaskExecutorOperator.class)
.withHost(executorHost)
- .killTask(new TaskInstanceKillRequest(taskInstance.getId()));
+ .killTask(TaskExecutorKillRequest.of(taskInstance.getId()));
if (killResponse.isSuccess()) {
log.info("Kill task {} on executor {} successfully", taskName, executorHost);
} else {
log.warn("Kill task {} on executor {} failed with response {}", taskName, executorHost, killResponse);
}
}
+
+ @Override
+ public void ackTaskExecutorLifecycleEvent(final ITaskExecutionRunnable taskExecutionRunnable,
+ final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck) {
+ final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance();
+ final String executorHost = taskInstance.getHost();
+ checkArgument(StringUtils.isNotEmpty(executorHost), "Executor host is empty");
+
+ Clients
+ .withService(IPhysicalTaskExecutorOperator.class)
+ .withHost(executorHost)
+ .ackPhysicalTaskExecutorLifecycleEvent(taskExecutorLifecycleEventAck);
+ }
+
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/TaskExecutorClient.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/TaskExecutorClient.java
index e83bbd5515..f564f3a1c9 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/TaskExecutorClient.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/client/TaskExecutorClient.java
@@ -23,7 +23,14 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskKillException;
import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskPauseException;
+import org.apache.dolphinscheduler.server.master.engine.exceptions.TaskReassignMasterHostException;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
+import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+
+import org.apache.commons.lang3.StringUtils;
+
+import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -31,6 +38,7 @@ import org.springframework.stereotype.Component;
/**
* The client of task executor, used to communicate with task executor.
*/
+@Slf4j
@Component
public class TaskExecutorClient implements ITaskExecutorClient {
@@ -41,7 +49,31 @@ public class TaskExecutorClient implements ITaskExecutorClient {
private PhysicalTaskExecutorClientDelegator physicalTaskExecutorClientDelegator;
@Override
- public void pause(ITaskExecutionRunnable taskExecutionRunnable) throws TaskPauseException {
+ public void dispatch(ITaskExecutionRunnable taskExecutionRunnable) throws TaskDispatchException {
+ try {
+ getTaskExecutorClientDelegator(taskExecutionRunnable).dispatch(taskExecutionRunnable);
+ } catch (TaskDispatchException taskDispatchException) {
+ throw taskDispatchException;
+ } catch (Exception ex) {
+ throw new TaskDispatchException("Dispatch task: " + taskExecutionRunnable.getName() + " to executor failed",
+ ex);
+ }
+ }
+
+ @Override
+ public boolean reassignWorkflowInstanceHost(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskReassignMasterHostException {
+ try {
+ return getTaskExecutorClientDelegator(taskExecutionRunnable)
+ .reassignMasterHost(taskExecutionRunnable);
+ } catch (Exception ex) {
+ throw new TaskReassignMasterHostException(
+ "Take over task: " + taskExecutionRunnable.getName() + " from executor failed",
+ ex);
+ }
+ }
+
+ @Override
+ public void pause(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskPauseException {
try {
getTaskExecutorClientDelegator(taskExecutionRunnable).pause(taskExecutionRunnable);
} catch (Exception ex) {
@@ -51,7 +83,7 @@ public class TaskExecutorClient implements ITaskExecutorClient {
}
@Override
- public void kill(ITaskExecutionRunnable taskExecutionRunnable) throws TaskKillException {
+ public void kill(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskKillException {
try {
getTaskExecutorClientDelegator(taskExecutionRunnable).kill(taskExecutionRunnable);
} catch (Exception ex) {
@@ -59,6 +91,23 @@ public class TaskExecutorClient implements ITaskExecutorClient {
}
}
+ @Override
+ public void ackTaskExecutorLifecycleEvent(
+ final ITaskExecutionRunnable taskExecutionRunnable,
+ final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck) {
+ try {
+ if (StringUtils.isEmpty(taskExecutionRunnable.getTaskInstance().getHost())) {
+ log.info("The task: {} is didn't dispatched to executor, skip ack taskExecutorLifecycleEventAck: {}",
+ taskExecutionRunnable.getName(), taskExecutorLifecycleEventAck);
+ return;
+ }
+ getTaskExecutorClientDelegator(taskExecutionRunnable)
+ .ackTaskExecutorLifecycleEvent(taskExecutionRunnable, taskExecutorLifecycleEventAck);
+ } catch (Exception ex) {
+ log.error("Send taskExecutorLifecycleEventAck: {} failed", taskExecutorLifecycleEventAck, ex);
+ }
+ }
+
private ITaskExecutorClientDelegator getTaskExecutorClientDelegator(final ITaskExecutionRunnable taskExecutionRunnable) {
final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance();
checkArgument(taskInstance != null, "taskType cannot be empty");
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/TaskLifecycleEventType.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/TaskLifecycleEventType.java
index 4e9cb4ad14..5ddcf13c89 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/TaskLifecycleEventType.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/TaskLifecycleEventType.java
@@ -38,6 +38,10 @@ public enum TaskLifecycleEventType implements ILifecycleEventType {
* The task instance is running at the target executor server.
*/
RUNNING,
+ /**
+ * The task instance's runtime context changed.
+ */
+ RUNTIME_CONTEXT_CHANGED,
/**
* Do Timeout strategy of the task instance.
*/
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRunningLifecycleEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRunningLifecycleEvent.java
index 571f23e4e6..e95d0c1e3f 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRunningLifecycleEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRunningLifecycleEvent.java
@@ -37,8 +37,6 @@ public class TaskRunningLifecycleEvent extends AbstractTaskLifecycleEvent {
private final String logPath;
- private final String runtimeContext;
-
private final Date startTime;
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRunningLifecycleEvent.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRuntimeContextChangedEvent.java
similarity index 84%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRunningLifecycleEvent.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRuntimeContextChangedEvent.java
index 571f23e4e6..681fb43f63 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRunningLifecycleEvent.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/event/TaskRuntimeContextChangedEvent.java
@@ -22,8 +22,6 @@ import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.AbstractT
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import java.util.Date;
-
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
@@ -31,27 +29,22 @@ import lombok.Getter;
@Getter
@Builder
@AllArgsConstructor
-public class TaskRunningLifecycleEvent extends AbstractTaskLifecycleEvent {
+public class TaskRuntimeContextChangedEvent extends AbstractTaskLifecycleEvent {
private final ITaskExecutionRunnable taskExecutionRunnable;
- private final String logPath;
-
private final String runtimeContext;
- private final Date startTime;
-
@Override
public ILifecycleEventType getEventType() {
- return TaskLifecycleEventType.RUNNING;
+ return TaskLifecycleEventType.RUNTIME_CONTEXT_CHANGED;
}
@Override
public String toString() {
return "TaskRunningLifecycleEvent{" +
"task=" + taskExecutionRunnable.getName() +
- ", logPath='" + logPath + '\'' +
- ", startTime=" + startTime +
+ ", runtimeContext=" + runtimeContext +
'}';
}
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/AbstractTaskLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/AbstractTaskLifecycleEventHandler.java
index 9c3b7a5f16..5396e9cdff 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/AbstractTaskLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/AbstractTaskLifecycleEventHandler.java
@@ -44,7 +44,7 @@ public abstract class AbstractTaskLifecycleEventHandler<T extends AbstractTaskLi
final TaskExecutionStatus state = taskExecutionRunnable.getTaskInstance().getState();
final ITaskStateAction taskStateAction = taskStateActionFactory.getTaskStateAction(state);
handle(taskStateAction, workflowExecutionRunnable, taskExecutionRunnable, event);
- log.info("Fired task {} LifecycleEvent[{}] with state {}",
+ log.info("Fired task {} {} with state {}",
taskExecutionRunnable.getName(),
event,
state.name());
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskDispatchedLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskDispatchedLifecycleEventHandler.java
index a0203af46e..10fbc3d2e9 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskDispatchedLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskDispatchedLifecycleEventHandler.java
@@ -18,11 +18,14 @@
package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler;
import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType;
+import org.apache.dolphinscheduler.server.master.engine.task.client.TaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskDispatchedLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.statemachine.ITaskStateAction;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorLifecycleEventType;
import lombok.extern.slf4j.Slf4j;
@@ -34,12 +37,23 @@ public class TaskDispatchedLifecycleEventHandler
extends
AbstractTaskLifecycleEventHandler<TaskDispatchedLifecycleEvent> {
+ private final TaskExecutorClient taskExecutorClient;
+
+ public TaskDispatchedLifecycleEventHandler(final TaskExecutorClient taskExecutorClient) {
+ this.taskExecutorClient = taskExecutorClient;
+ }
+
@Override
public void handle(final ITaskStateAction taskStateAction,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskDispatchedLifecycleEvent taskDispatchedEvent) {
taskStateAction.dispatchedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskDispatchedEvent);
+ taskExecutorClient.ackTaskExecutorLifecycleEvent(
+ taskExecutionRunnable,
+ new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck(
+ taskExecutionRunnable.getId(),
+ TaskExecutorLifecycleEventType.DISPATCHED));
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskFailedLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskFailedLifecycleEventHandler.java
index c83ac98946..2c521f4086 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskFailedLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskFailedLifecycleEventHandler.java
@@ -18,23 +18,37 @@
package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler;
import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType;
+import org.apache.dolphinscheduler.server.master.engine.task.client.TaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskFailedLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.statemachine.ITaskStateAction;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorLifecycleEventType;
import org.springframework.stereotype.Component;
@Component
public class TaskFailedLifecycleEventHandler extends AbstractTaskLifecycleEventHandler<TaskFailedLifecycleEvent> {
+ private final TaskExecutorClient taskExecutorClient;
+
+ public TaskFailedLifecycleEventHandler(final TaskExecutorClient taskExecutorClient) {
+ this.taskExecutorClient = taskExecutorClient;
+ }
+
@Override
public void handle(final ITaskStateAction taskStateAction,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskFailedLifecycleEvent event) {
taskStateAction.failedEventAction(workflowExecutionRunnable, taskExecutionRunnable, event);
+ taskExecutorClient.ackTaskExecutorLifecycleEvent(
+ taskExecutionRunnable,
+ new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck(
+ taskExecutionRunnable.getId(),
+ TaskExecutorLifecycleEventType.FAILED));
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskKilledLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskKilledLifecycleEventHandler.java
index b84a770dde..c2a686cc45 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskKilledLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskKilledLifecycleEventHandler.java
@@ -18,23 +18,37 @@
package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler;
import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType;
+import org.apache.dolphinscheduler.server.master.engine.task.client.TaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskKilledLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.statemachine.ITaskStateAction;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorLifecycleEventType;
import org.springframework.stereotype.Component;
@Component
public class TaskKilledLifecycleEventHandler extends AbstractTaskLifecycleEventHandler<TaskKilledLifecycleEvent> {
+ private final TaskExecutorClient taskExecutorClient;
+
+ public TaskKilledLifecycleEventHandler(final TaskExecutorClient taskExecutorClient) {
+ this.taskExecutorClient = taskExecutorClient;
+ }
+
@Override
public void handle(final ITaskStateAction taskStateAction,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskKilledLifecycleEvent taskKilledEvent) {
taskStateAction.killedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskKilledEvent);
+ taskExecutorClient.ackTaskExecutorLifecycleEvent(
+ taskExecutionRunnable,
+ new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck(
+ taskExecutionRunnable.getId(),
+ TaskExecutorLifecycleEventType.KILLED));
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskPausedLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskPausedLifecycleEventHandler.java
index 49811b90a1..c3951f3b82 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskPausedLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskPausedLifecycleEventHandler.java
@@ -18,23 +18,37 @@
package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler;
import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType;
+import org.apache.dolphinscheduler.server.master.engine.task.client.TaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskPausedLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.statemachine.ITaskStateAction;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorLifecycleEventType;
import org.springframework.stereotype.Component;
@Component
public class TaskPausedLifecycleEventHandler extends AbstractTaskLifecycleEventHandler<TaskPausedLifecycleEvent> {
+ private final TaskExecutorClient taskExecutorClient;
+
+ public TaskPausedLifecycleEventHandler(final TaskExecutorClient taskExecutorClient) {
+ this.taskExecutorClient = taskExecutorClient;
+ }
+
@Override
public void handle(final ITaskStateAction taskStateAction,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskPausedLifecycleEvent event) {
taskStateAction.pausedEventAction(workflowExecutionRunnable, taskExecutionRunnable, event);
+ taskExecutorClient.ackTaskExecutorLifecycleEvent(
+ taskExecutionRunnable,
+ new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck(
+ taskExecutionRunnable.getId(),
+ TaskExecutorLifecycleEventType.PAUSED));
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskRunningLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskRunningLifecycleEventHandler.java
index 4bd5d1ee20..c0899dcf01 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskRunningLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskRunningLifecycleEventHandler.java
@@ -18,11 +18,14 @@
package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler;
import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType;
+import org.apache.dolphinscheduler.server.master.engine.task.client.ITaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRunningLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.statemachine.ITaskStateAction;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorLifecycleEventType;
import lombok.extern.slf4j.Slf4j;
@@ -32,12 +35,23 @@ import org.springframework.stereotype.Component;
@Component
public class TaskRunningLifecycleEventHandler extends AbstractTaskLifecycleEventHandler<TaskRunningLifecycleEvent> {
+ private final ITaskExecutorClient taskExecutorClient;
+
+ public TaskRunningLifecycleEventHandler(final ITaskExecutorClient taskExecutorClient) {
+ this.taskExecutorClient = taskExecutorClient;
+ }
+
@Override
public void handle(final ITaskStateAction taskStateAction,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskRunningLifecycleEvent taskRunningEvent) {
taskStateAction.startedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskRunningEvent);
+ taskExecutorClient.ackTaskExecutorLifecycleEvent(
+ taskExecutionRunnable,
+ new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck(
+ taskExecutionRunnable.getId(),
+ TaskExecutorLifecycleEventType.RUNNING));
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskDispatchedLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskRuntimeContextChangedLifecycleEventHandler.java
similarity index 59%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskDispatchedLifecycleEventHandler.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskRuntimeContextChangedLifecycleEventHandler.java
index a0203af46e..54698cf90e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskDispatchedLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskRuntimeContextChangedLifecycleEventHandler.java
@@ -18,11 +18,14 @@
package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler;
import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType;
+import org.apache.dolphinscheduler.server.master.engine.task.client.ITaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
-import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskDispatchedLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRuntimeContextChangedEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.statemachine.ITaskStateAction;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorLifecycleEventType;
import lombok.extern.slf4j.Slf4j;
@@ -30,20 +33,33 @@ import org.springframework.stereotype.Component;
@Slf4j
@Component
-public class TaskDispatchedLifecycleEventHandler
+public class TaskRuntimeContextChangedLifecycleEventHandler
extends
- AbstractTaskLifecycleEventHandler<TaskDispatchedLifecycleEvent> {
+ AbstractTaskLifecycleEventHandler<TaskRuntimeContextChangedEvent> {
+
+ private final ITaskExecutorClient taskExecutorClient;
+
+ public TaskRuntimeContextChangedLifecycleEventHandler(final ITaskExecutorClient taskExecutorClient) {
+ this.taskExecutorClient = taskExecutorClient;
+ }
@Override
public void handle(final ITaskStateAction taskStateAction,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
final ITaskExecutionRunnable taskExecutionRunnable,
- final TaskDispatchedLifecycleEvent taskDispatchedEvent) {
- taskStateAction.dispatchedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskDispatchedEvent);
+ final TaskRuntimeContextChangedEvent event) {
+ taskStateAction.runtimeContextChangedEventAction(workflowExecutionRunnable, taskExecutionRunnable, event);
+
+ taskExecutorClient.ackTaskExecutorLifecycleEvent(
+ taskExecutionRunnable,
+ new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck(
+ taskExecutionRunnable.getId(),
+ TaskExecutorLifecycleEventType.RUNTIME_CONTEXT_CHANGE));
}
@Override
public ILifecycleEventType matchEventType() {
- return TaskLifecycleEventType.DISPATCHED;
+ return TaskLifecycleEventType.RUNTIME_CONTEXT_CHANGED;
}
+
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java
index b79ec46bf8..aa4299a540 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskSuccessLifecycleEventHandler.java
@@ -18,23 +18,37 @@
package org.apache.dolphinscheduler.server.master.engine.task.lifecycle.handler;
import org.apache.dolphinscheduler.server.master.engine.ILifecycleEventType;
+import org.apache.dolphinscheduler.server.master.engine.task.client.TaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.TaskLifecycleEventType;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.statemachine.ITaskStateAction;
import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorLifecycleEventType;
import org.springframework.stereotype.Component;
@Component
public class TaskSuccessLifecycleEventHandler extends AbstractTaskLifecycleEventHandler<TaskSuccessLifecycleEvent> {
+ private final TaskExecutorClient taskExecutorClient;
+
+ public TaskSuccessLifecycleEventHandler(final TaskExecutorClient taskExecutorClient) {
+ this.taskExecutorClient = taskExecutorClient;
+ }
+
@Override
public void handle(final ITaskStateAction taskStateAction,
final IWorkflowExecutionRunnable workflowExecutionRunnable,
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskSuccessLifecycleEvent taskSuccessEvent) {
taskStateAction.succeedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskSuccessEvent);
+ taskExecutorClient.ackTaskExecutorLifecycleEvent(
+ taskExecutionRunnable,
+ new ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck(
+ taskExecutionRunnable.getId(),
+ TaskExecutorLifecycleEventType.SUCCESS));
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskTimeoutLifecycleEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskTimeoutLifecycleEventHandler.java
index dd4d5546dd..d406168c07 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskTimeoutLifecycleEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/lifecycle/handler/TaskTimeoutLifecycleEventHandler.java
@@ -33,15 +33,17 @@ import org.apache.dolphinscheduler.service.alert.WorkflowAlertManager;
import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class TaskTimeoutLifecycleEventHandler extends AbstractTaskLifecycleEventHandler<TaskTimeoutLifecycleEvent> {
- @Autowired
- private WorkflowAlertManager workflowAlertManager;
+ private final WorkflowAlertManager workflowAlertManager;
+
+ public TaskTimeoutLifecycleEventHandler(final WorkflowAlertManager workflowAlertManager) {
+ this.workflowAlertManager = workflowAlertManager;
+ }
@Override
public void handle(final ITaskStateAction taskStateAction,
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/ITaskExecutionRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/ITaskExecutionRunnable.java
index ac01b60873..d0e0ad8d9d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/ITaskExecutionRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/ITaskExecutionRunnable.java
@@ -32,6 +32,14 @@ public interface ITaskExecutionRunnable
extends
Comparable<ITaskExecutionRunnable> {
+ /**
+ * Get the task instance id.
+ * <p> Need to know the id might change since the task instance might be regenerated.
+ */
+ default int getId() {
+ return getTaskInstance().getId();
+ }
+
default String getName() {
return getTaskDefinition().getName();
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnable.java
index 1c364a7e22..9f45490a72 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnable.java
@@ -25,21 +25,16 @@ import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition;
import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
-import org.apache.dolphinscheduler.extract.base.client.Clients;
-import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.worker.transportor.TakeOverTaskRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TakeOverTaskResponse;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
-import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.engine.WorkflowEventBus;
import org.apache.dolphinscheduler.server.master.engine.graph.IWorkflowExecutionGraph;
+import org.apache.dolphinscheduler.server.master.engine.task.client.ITaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskKillLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskPauseLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskStartLifecycleEvent;
import org.apache.dolphinscheduler.server.master.runner.TaskExecutionContextFactory;
-import org.apache.commons.lang3.StringUtils;
+import javax.annotation.Nullable;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -62,7 +57,7 @@ public class TaskExecutionRunnable implements ITaskExecutionRunnable {
@Getter
private final WorkflowInstance workflowInstance;
@Getter
- private TaskInstance taskInstance;
+ private @Nullable TaskInstance taskInstance;
@Getter
private final TaskDefinition taskDefinition;
@Getter
@@ -159,24 +154,8 @@ public class TaskExecutionRunnable implements ITaskExecutionRunnable {
private boolean takeOverTaskFromExecutor() {
checkState(isTaskInstanceInitialized(), "The task instance is null, can't take over from executor.");
- if (TaskTypeUtils.isLogicTask(taskInstance.getTaskType())) {
- return false;
- }
- if (StringUtils.isEmpty(taskInstance.getHost())) {
- log.debug("Task: {} host is empty, cannot take over the task from executor(This is normal case).",
- taskInstance.getName());
- return false;
- }
try {
- final TakeOverTaskRequest takeOverTaskRequest = TakeOverTaskRequest.builder()
- .taskInstanceId(taskInstance.getId())
- .workflowHost(applicationContext.getBean(MasterConfig.class).getMasterAddress())
- .build();
- final TakeOverTaskResponse takeOverTaskResponse = Clients
- .withService(ITaskInstanceOperator.class)
- .withHost(taskInstance.getHost())
- .takeOverTask(takeOverTaskRequest);
- return takeOverTaskResponse.isSuccess();
+ return applicationContext.getBean(ITaskExecutorClient.class).reassignWorkflowInstanceHost(this);
} catch (Exception ex) {
log.warn("Take over task: {} failed", taskInstance.getName(), ex);
return false;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnableBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnableBuilder.java
index 24d1dbd04e..ad213de495 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnableBuilder.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/runnable/TaskExecutionRunnableBuilder.java
@@ -25,6 +25,8 @@ import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
import org.apache.dolphinscheduler.server.master.engine.WorkflowEventBus;
import org.apache.dolphinscheduler.server.master.engine.graph.IWorkflowExecutionGraph;
+import javax.annotation.Nullable;
+
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
@@ -41,7 +43,7 @@ public class TaskExecutionRunnableBuilder {
private final Project project;
private final WorkflowInstance workflowInstance;
private final TaskDefinition taskDefinition;
- private final TaskInstance taskInstance;
+ private final @Nullable TaskInstance taskInstance;
private final WorkflowEventBus workflowEventBus;
private final ApplicationContext applicationContext;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/AbstractTaskStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/AbstractTaskStateAction.java
index 1de7181746..867a6edeec 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/AbstractTaskStateAction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/AbstractTaskStateAction.java
@@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.Tas
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskPausedLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRetryLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRunningLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRuntimeContextChangedEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.TaskInstanceFactories;
@@ -110,15 +111,23 @@ public abstract class AbstractTaskStateAction implements ITaskStateAction {
taskInstanceDao.updateById(taskInstance);
}
+ @Override
+ public void runtimeContextChangedEventAction(final IWorkflowExecutionRunnable workflowExecutionRunnable,
+ final ITaskExecutionRunnable taskExecutionRunnable,
+ final TaskRuntimeContextChangedEvent taskRuntimeContextChangedEvent) {
+ final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance();
+ if (StringUtils.isNotEmpty(taskRuntimeContextChangedEvent.getRuntimeContext())) {
+ taskInstance.setAppLink(taskRuntimeContextChangedEvent.getRuntimeContext());
+ }
+ taskInstanceDao.updateById(taskInstance);
+ }
+
protected void persistentTaskInstanceStartedEventToDB(final ITaskExecutionRunnable taskExecutionRunnable,
final TaskRunningLifecycleEvent taskRunningEvent) {
final TaskInstance taskInstance = taskExecutionRunnable.getTaskInstance();
taskInstance.setState(TaskExecutionStatus.RUNNING_EXECUTION);
taskInstance.setStartTime(taskRunningEvent.getStartTime());
taskInstance.setLogPath(taskRunningEvent.getLogPath());
- if (StringUtils.isNotEmpty(taskRunningEvent.getRuntimeContext())) {
- taskInstance.setAppLink(taskRunningEvent.getRuntimeContext());
- }
taskInstanceDao.updateById(taskInstance);
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/ITaskStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/ITaskStateAction.java
index 432c7d6d7a..ac4debfdbe 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/ITaskStateAction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/ITaskStateAction.java
@@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.Tas
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskPausedLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRetryLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRunningLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRuntimeContextChangedEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskStartLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
@@ -66,6 +67,14 @@ public interface ITaskStateAction {
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskRunningLifecycleEvent taskRunningEvent);
+ /**
+ * Perform the necessary actions when the task in a certain state receive a {@link TaskRuntimeContextChangedEvent}.
+ * <p> This method is called when the master receive task runtime context changed event from executor.
+ */
+ void runtimeContextChangedEventAction(final IWorkflowExecutionRunnable workflowExecutionRunnable,
+ final ITaskExecutionRunnable taskExecutionRunnable,
+ final TaskRuntimeContextChangedEvent taskRuntimeContextChangedEvent);
+
/**
* Perform the necessary actions when the task in a certain state receive a {@link TaskRetryLifecycleEvent}.
* <p> This method is called when the task need to retry.
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskDispatchStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskDispatchStateAction.java
index 0b9d174f8a..acb31bb087 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskDispatchStateAction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskDispatchStateAction.java
@@ -94,7 +94,7 @@ public class TaskDispatchStateAction extends AbstractTaskStateAction {
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskPauseLifecycleEvent taskPauseEvent) {
throwExceptionIfStateIsNotMatch(taskExecutionRunnable);
- taskExecutorClient.kill(taskExecutionRunnable);
+ taskExecutorClient.pause(taskExecutionRunnable);
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskSubmittedStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskSubmittedStateAction.java
index aca7b77ffd..184e5d4c67 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskSubmittedStateAction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/task/statemachine/TaskSubmittedStateAction.java
@@ -169,7 +169,7 @@ public class TaskSubmittedStateAction extends AbstractTaskStateAction {
final ITaskExecutionRunnable taskExecutionRunnable,
final TaskFailedLifecycleEvent taskFailedEvent) {
throwExceptionIfStateIsNotMatch(taskExecutionRunnable);
- logWarningIfCannotDoAction(taskExecutionRunnable, taskFailedEvent);
+ super.failedEventAction(workflowExecutionRunnable, taskExecutionRunnable, taskFailedEvent);
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java
index 7ffa5767f0..8b78903a5a 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/engine/workflow/statemachine/AbstractWorkflowStateAction.java
@@ -106,7 +106,6 @@ public abstract class AbstractWorkflowStateAction implements IWorkflowStateActio
final WorkflowExecutionStatus workflowExecutionStatus) {
final WorkflowInstance workflowInstance = workflowExecutionRunnable.getWorkflowInstance();
workflowInstance.setEndTime(new Date());
- workflowInstance.setState(workflowExecutionStatus);
transformWorkflowInstanceState(workflowExecutionRunnable, workflowExecutionStatus);
workflowExecutionRunnable.getWorkflowEventBus()
.publish(WorkflowFinalizeLifecycleEvent.of(workflowExecutionRunnable));
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceDispatchOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceDispatchOperationFunction.java
deleted file mode 100644
index 57b7ae3976..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceDispatchOperationFunction.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.rpc;
-
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutionContextHolder;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutor;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutorFactoryBuilder;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutorThreadPoolManager;
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class LogicITaskInstanceDispatchOperationFunction
- implements
- ITaskInstanceOperationFunction<LogicTaskDispatchRequest, LogicTaskDispatchResponse> {
-
- @Autowired
- private MasterTaskExecutorFactoryBuilder masterTaskExecutorFactoryBuilder;
-
- @Autowired
- private MasterTaskExecutorThreadPoolManager masterTaskExecutorThreadPool;
-
- @Autowired
- private LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager;
-
- @Override
- public LogicTaskDispatchResponse operate(LogicTaskDispatchRequest taskDispatchRequest) {
- log.info("Received dispatchLogicTask request: {}", taskDispatchRequest);
- TaskExecutionContext taskExecutionContext = taskDispatchRequest.getTaskExecutionContext();
- try {
- final int taskInstanceId = taskExecutionContext.getTaskInstanceId();
- final int workflowInstanceId = taskExecutionContext.getWorkflowInstanceId();
- final String taskInstanceName = taskExecutionContext.getTaskName();
-
- taskExecutionContext.setLogPath(LogUtils.getTaskInstanceLogFullPath(taskExecutionContext));
-
- LogUtils.setWorkflowAndTaskInstanceIDMDC(workflowInstanceId, taskInstanceId);
- LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath());
-
- MasterTaskExecutionContextHolder.putTaskExecutionContext(taskExecutionContext);
-
- final MasterTaskExecutor masterTaskExecutor = masterTaskExecutorFactoryBuilder
- .createMasterTaskExecutorFactory(taskExecutionContext.getTaskType())
- .createMasterTaskExecutor(taskExecutionContext);
- if (masterTaskExecutorThreadPool.submitMasterTaskExecutor(masterTaskExecutor)) {
- log.info("Submit LogicTask: {} to MasterTaskExecutorThreadPool success", taskInstanceName);
- return LogicTaskDispatchResponse.success(taskInstanceId);
- } else {
- log.error("Submit LogicTask: {} to MasterTaskExecutorThreadPool failed", taskInstanceName);
- return LogicTaskDispatchResponse.failed(taskInstanceId, "MasterTaskExecutorThreadPool is full");
- }
- } finally {
- LogUtils.removeWorkflowAndTaskInstanceIdMDC();
- LogUtils.removeTaskInstanceLogFullPathMDC();
- }
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceKillOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceKillOperationFunction.java
deleted file mode 100644
index 59e18993da..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstanceKillOperationFunction.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.rpc;
-
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse;
-import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutor;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutorHolder;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutorThreadPoolManager;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class LogicITaskInstanceKillOperationFunction
- implements
- ITaskInstanceOperationFunction<LogicTaskKillRequest, LogicTaskKillResponse> {
-
- @Autowired
- private MasterTaskExecutorThreadPoolManager masterTaskExecutorThreadPool;
-
- @Override
- public LogicTaskKillResponse operate(LogicTaskKillRequest taskKillRequest) {
- final int taskInstanceId = taskKillRequest.getTaskInstanceId();
- try {
- LogUtils.setTaskInstanceIdMDC(taskKillRequest.getTaskInstanceId());
- log.info("Received killLogicTask request: {}", taskKillRequest);
- final MasterTaskExecutor masterTaskExecutor =
- MasterTaskExecutorHolder.getMasterTaskExecutor(taskInstanceId);
- if (masterTaskExecutor == null) {
- log.error("Cannot find the MasterTaskExecuteRunnable, this task may already been killed");
- return LogicTaskKillResponse.fail("Cannot find the MasterTaskExecuteRunnable");
- }
- try {
- masterTaskExecutor.cancelTask();
- return LogicTaskKillResponse.success();
- } catch (MasterTaskExecuteException e) {
- log.error("Cancel MasterTaskExecuteRunnable failed ", e);
- return LogicTaskKillResponse.fail("Cancel MasterTaskExecuteRunnable failed: " + e.getMessage());
- }
- } finally {
- LogUtils.removeTaskInstanceIdMDC();
- }
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstancePauseOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstancePauseOperationFunction.java
deleted file mode 100644
index 12b45e6fe3..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicITaskInstancePauseOperationFunction.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.rpc;
-
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutor;
-import org.apache.dolphinscheduler.server.master.runner.execute.MasterTaskExecutorHolder;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class LogicITaskInstancePauseOperationFunction
- implements
- ITaskInstanceOperationFunction<LogicTaskPauseRequest, LogicTaskPauseResponse> {
-
- @Override
- public LogicTaskPauseResponse operate(LogicTaskPauseRequest taskPauseRequest) {
- try {
- LogUtils.setTaskInstanceIdMDC(taskPauseRequest.getTaskInstanceId());
- final MasterTaskExecutor masterTaskExecutor =
- MasterTaskExecutorHolder.getMasterTaskExecutor(taskPauseRequest.getTaskInstanceId());
- if (masterTaskExecutor == null) {
- log.info("Cannot find the MasterTaskExecuteRunnable");
- return LogicTaskPauseResponse.fail("Cannot find the MasterTaskExecuteRunnable");
- }
- final TaskExecutionContext taskExecutionContext = masterTaskExecutor.getTaskExecutionContext();
- LogUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getWorkflowInstanceId(),
- taskExecutionContext.getTaskInstanceId());
- masterTaskExecutor.pauseTask();
- return LogicTaskPauseResponse.success();
- } catch (MasterTaskExecuteException e) {
- log.error("Pause MasterTaskExecuteRunnable failed", e);
- return LogicTaskPauseResponse.fail("Pause MasterTaskExecuteRunnable failed: " + e.getMessage());
- } finally {
- LogUtils.removeTaskInstanceIdMDC();
- LogUtils.removeWorkflowAndTaskInstanceIdMDC();
- }
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskExecutorOperatorImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskExecutorOperatorImpl.java
new file mode 100644
index 0000000000..e3fbade805
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskExecutorOperatorImpl.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.rpc;
+
+import org.apache.dolphinscheduler.extract.master.ILogicTaskExecutorOperator;
+import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
+import org.apache.dolphinscheduler.server.master.engine.executor.LogicTaskEngineDelegator;
+import org.apache.dolphinscheduler.task.executor.eventbus.ITaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.task.executor.log.TaskExecutorMDCUtils;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorDispatchResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorKillResponse;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseRequest;
+import org.apache.dolphinscheduler.task.executor.operations.TaskExecutorPauseResponse;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+public class LogicTaskExecutorOperatorImpl implements ILogicTaskExecutorOperator {
+
+ @Autowired
+ private LogicTaskEngineDelegator logicTaskEngineDelegator;
+
+ @Override
+ public TaskExecutorDispatchResponse dispatchTask(final TaskExecutorDispatchRequest taskExecutorDispatchRequest) {
+ final TaskExecutionContext taskExecutionContext = taskExecutorDispatchRequest.getTaskExecutionContext();
+ try (
+ final TaskExecutorMDCUtils.MDCAutoClosable ignore =
+ TaskExecutorMDCUtils.logWithMDC(taskExecutionContext.getTaskInstanceId())) {
+ log.info("Receive {}", taskExecutorDispatchRequest);
+ try {
+ logicTaskEngineDelegator.dispatchLogicTask(taskExecutionContext);
+ log.info("Handle {} success", taskExecutorDispatchRequest);
+ return TaskExecutorDispatchResponse.success();
+ } catch (Throwable throwable) {
+ log.error("Handle {} failed", taskExecutorDispatchRequest, throwable);
+ return TaskExecutorDispatchResponse.failed(ExceptionUtils.getMessage(throwable));
+ }
+ }
+ }
+
+ @Override
+ public TaskExecutorPauseResponse pauseTask(final TaskExecutorPauseRequest taskPauseRequest) {
+ final int taskInstanceId = taskPauseRequest.getTaskInstanceId();
+ try (final TaskExecutorMDCUtils.MDCAutoClosable ignore = TaskExecutorMDCUtils.logWithMDC(taskInstanceId)) {
+ log.info("Receive {}", taskPauseRequest);
+ try {
+ logicTaskEngineDelegator.pauseLogicTask(taskInstanceId);
+ log.info("Handle {} success", taskPauseRequest);
+ return TaskExecutorPauseResponse.success();
+ } catch (Throwable throwable) {
+ log.error("Handle {} failed", taskPauseRequest, throwable);
+ return TaskExecutorPauseResponse.fail(ExceptionUtils.getMessage(throwable));
+ }
+ }
+ }
+
+ @Override
+ public void ackTaskExecutorLifecycleEvent(final ITaskExecutorLifecycleEventReporter.TaskExecutorLifecycleEventAck taskExecutorLifecycleEventAck) {
+ final int taskExecutorId = taskExecutorLifecycleEventAck.getTaskExecutorId();
+ try (final TaskExecutorMDCUtils.MDCAutoClosable ignore = TaskExecutorMDCUtils.logWithMDC(taskExecutorId)) {
+ log.info("Receive : {}", taskExecutorLifecycleEventAck);
+ logicTaskEngineDelegator.ackLogicTaskExecutionEvent(taskExecutorLifecycleEventAck);
+ }
+ }
+
+ @Override
+ public TaskExecutorKillResponse killTask(final TaskExecutorKillRequest taskKillRequest) {
+ final int taskInstanceId = taskKillRequest.getTaskInstanceId();
+ try (final TaskExecutorMDCUtils.MDCAutoClosable ignore = TaskExecutorMDCUtils.logWithMDC(taskInstanceId)) {
+ log.info("Receive {}", taskKillRequest);
+ try {
+ logicTaskEngineDelegator.killLogicTask(taskInstanceId);
+ log.info("Handle {} success", taskKillRequest);
+ return TaskExecutorKillResponse.success();
+ } catch (Throwable throwable) {
+ log.error("Handle {} failed", taskKillRequest, throwable);
+ return TaskExecutorKillResponse.fail(ExceptionUtils.getMessage(throwable));
+ }
+ }
+ }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java
deleted file mode 100644
index de4095831e..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.rpc;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class LogicTaskInstanceOperationFunctionManager {
-
- @Autowired
- private LogicITaskInstanceDispatchOperationFunction logicITaskInstanceDispatchOperationFunction;
-
- @Autowired
- private LogicITaskInstanceKillOperationFunction logicITaskInstanceKillOperationFunction;
-
- @Autowired
- private LogicITaskInstancePauseOperationFunction logicITaskInstancePauseOperationFunction;
-
- public LogicITaskInstanceDispatchOperationFunction getLogicTaskInstanceDispatchOperationFunction() {
- return logicITaskInstanceDispatchOperationFunction;
- }
-
- public LogicITaskInstanceKillOperationFunction getLogicTaskInstanceKillOperationFunction() {
- return logicITaskInstanceKillOperationFunction;
- }
-
- public LogicITaskInstancePauseOperationFunction getLogicTaskInstancePauseOperationFunction() {
- return logicITaskInstancePauseOperationFunction;
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java
deleted file mode 100644
index 39446bd89d..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.rpc;
-
-import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-@Slf4j
-@Service
-public class LogicTaskInstanceOperatorImpl implements ILogicTaskInstanceOperator {
-
- @Autowired
- private LogicTaskInstanceOperationFunctionManager logicTaskInstanceOperationFunctionManager;
-
- @Override
- public LogicTaskDispatchResponse dispatchLogicTask(LogicTaskDispatchRequest taskDispatchRequest) {
- return logicTaskInstanceOperationFunctionManager.getLogicTaskInstanceDispatchOperationFunction()
- .operate(taskDispatchRequest);
- }
-
- @Override
- public LogicTaskKillResponse killLogicTask(LogicTaskKillRequest taskKillRequest) {
- return logicTaskInstanceOperationFunctionManager.getLogicTaskInstanceKillOperationFunction()
- .operate(taskKillRequest);
- }
-
- @Override
- public LogicTaskPauseResponse pauseLogicTask(LogicTaskPauseRequest taskPauseRequest) {
- return logicTaskInstanceOperationFunctionManager.getLogicTaskInstancePauseOperationFunction()
- .operate(taskPauseRequest);
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java
index 0fa7cbffd9..98e7461f50 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/MasterLogServiceImpl.java
@@ -20,14 +20,11 @@ package org.apache.dolphinscheduler.server.master.rpc;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.LogUtils;
import org.apache.dolphinscheduler.extract.common.ILogService;
-import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdRequest;
-import org.apache.dolphinscheduler.extract.common.transportor.GetAppIdResponse;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadRequest;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogFileDownloadResponse;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryRequest;
import org.apache.dolphinscheduler.extract.common.transportor.TaskInstanceLogPageQueryResponse;
-import java.util.Collections;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
@@ -58,11 +55,6 @@ public class MasterLogServiceImpl implements ILogService {
return new TaskInstanceLogPageQueryResponse(logContent);
}
- @Override
- public GetAppIdResponse getAppId(GetAppIdRequest getAppIdRequest) {
- return new GetAppIdResponse(Collections.emptyList());
- }
-
@Override
public void removeTaskInstanceLog(String taskInstanceLogAbsolutePath) {
FileUtils.deleteFile(taskInstanceLogAbsolutePath);
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutionEventListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutionEventListenerImpl.java
deleted file mode 100644
index b788337033..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutionEventListenerImpl.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.rpc;
-
-import org.apache.dolphinscheduler.extract.base.client.Clients;
-import org.apache.dolphinscheduler.extract.master.ITaskExecutionEventListener;
-import org.apache.dolphinscheduler.extract.master.transportor.ITaskExecutionEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionDispatchEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionFailedEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionKilledEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionPausedEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionRunningEvent;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionSuccessEvent;
-import org.apache.dolphinscheduler.extract.worker.ITaskInstanceExecutionEventAckListener;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionFailedEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionKilledEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionPausedEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskExecutionSuccessEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionDispatchedEventAck;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceExecutionRunningEventAck;
-import org.apache.dolphinscheduler.plugin.task.api.utils.TaskTypeUtils;
-import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
-import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskDispatchedLifecycleEvent;
-import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskFailedLifecycleEvent;
-import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskKilledLifecycleEvent;
-import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskPausedLifecycleEvent;
-import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRunningLifecycleEvent;
-import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent;
-import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-
-import java.util.Date;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-@Slf4j
-@Service
-public class TaskExecutionEventListenerImpl implements ITaskExecutionEventListener {
-
- @Autowired
- private IWorkflowRepository IWorkflowRepository;
-
- @Override
- public void onTaskInstanceDispatched(TaskExecutionDispatchEvent taskExecutionDispatchEvent) {
- final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutionDispatchEvent);
- final TaskDispatchedLifecycleEvent taskDispatchedLifecycleEvent = TaskDispatchedLifecycleEvent.builder()
- .taskExecutionRunnable(taskExecutionRunnable)
- .executorHost(taskExecutionDispatchEvent.getTaskInstanceHost())
- .build();
-
- taskExecutionRunnable.getWorkflowEventBus().publish(taskDispatchedLifecycleEvent);
- // Once the master receive the event, then will send ack to the worker
- // This means the worker will not retry to send the event to the master
- // So once the master failover and we take over the task instance success, then we should fetch the latest task
- // instance state.
- // The logic task doesn't need to send ack
- if (!TaskTypeUtils.isLogicTask(taskExecutionRunnable.getTaskDefinition().getTaskType())) {
- Clients
- .withService(ITaskInstanceExecutionEventAckListener.class)
- .withHost(taskExecutionDispatchEvent.getTaskInstanceHost())
- .handleTaskInstanceDispatchedEventAck(
- TaskInstanceExecutionDispatchedEventAck
- .success(taskExecutionDispatchEvent.getTaskInstanceId()));
- }
- }
-
- @Override
- public void onTaskInstanceExecutionRunning(TaskExecutionRunningEvent taskExecutionRunningEvent) {
- final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutionRunningEvent);
- final TaskRunningLifecycleEvent taskRunningEvent = TaskRunningLifecycleEvent.builder()
- .taskExecutionRunnable(taskExecutionRunnable)
- .startTime(new Date(taskExecutionRunningEvent.getStartTime()))
- .logPath(taskExecutionRunningEvent.getLogPath())
- .runtimeContext(taskExecutionRunningEvent.getAppIds())
- .build();
-
- taskExecutionRunnable.getWorkflowEventBus().publish(taskRunningEvent);
- // Once the master receive the event, then will send ack to the worker
- // This means the worker will not retry to send the event to the master
- // So once the master failover and we take over the task instance success, then we should fetch the latest task
- // instance state.
- // The logic task doesn't need to send ack
- if (!TaskTypeUtils.isLogicTask(taskExecutionRunnable.getTaskDefinition().getTaskType())) {
- Clients
- .withService(ITaskInstanceExecutionEventAckListener.class)
- .withHost(taskExecutionRunningEvent.getTaskInstanceHost())
- .handleTaskInstanceExecutionRunningEventAck(
- TaskInstanceExecutionRunningEventAck
- .success(taskExecutionRunningEvent.getTaskInstanceId()));
- }
- }
-
- @Override
- public void onTaskInstanceExecutionSuccess(final TaskExecutionSuccessEvent taskExecutionSuccessEvent) {
- final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutionSuccessEvent);
- final TaskSuccessLifecycleEvent taskSuccessEvent = TaskSuccessLifecycleEvent.builder()
- .taskExecutionRunnable(taskExecutionRunnable)
- .endTime(new Date(taskExecutionSuccessEvent.getEndTime()))
- .varPool(taskExecutionSuccessEvent.getVarPool())
- .build();
- taskExecutionRunnable.getWorkflowEventBus().publish(taskSuccessEvent);
- // Once the master receive the event, then will send ack to the worker
- // This means the worker will not retry to send the event to the master
- // So once the master failover and we take over the task instance success, then we should fetch the latest task
- // instance state.
- if (!TaskTypeUtils.isLogicTask(taskExecutionRunnable.getTaskDefinition().getTaskType())) {
- Clients
- .withService(ITaskInstanceExecutionEventAckListener.class)
- .withHost(taskExecutionSuccessEvent.getTaskInstanceHost())
- .handleTaskExecutionSuccessEventAck(
- TaskExecutionSuccessEventAck.success(taskExecutionSuccessEvent.getTaskInstanceId()));
- }
- }
-
- @Override
- public void onTaskInstanceExecutionFailed(TaskExecutionFailedEvent taskExecutionFailedEvent) {
- final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutionFailedEvent);
- final TaskFailedLifecycleEvent taskFailedEvent = TaskFailedLifecycleEvent.builder()
- .taskExecutionRunnable(taskExecutionRunnable)
- .endTime(new Date(taskExecutionFailedEvent.getEndTime()))
- .build();
- taskExecutionRunnable.getWorkflowEventBus().publish(taskFailedEvent);
- // Once the master receive the event, then will send ack to the worker
- // This means the worker will not retry to send the event to the master
- // So once the master failover and we take over the task instance success, then we should fetch the latest task
- // instance state.
- if (!TaskTypeUtils.isLogicTask(taskExecutionRunnable.getTaskDefinition().getTaskType())) {
- Clients
- .withService(ITaskInstanceExecutionEventAckListener.class)
- .withHost(taskExecutionFailedEvent.getTaskInstanceHost())
- .handleTaskExecutionFailedEventAck(
- TaskExecutionFailedEventAck.success(taskExecutionFailedEvent.getTaskInstanceId()));
- }
- }
-
- @Override
- public void onTaskInstanceExecutionKilled(TaskExecutionKilledEvent taskExecutionKilledEvent) {
- final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutionKilledEvent);
- final TaskKilledLifecycleEvent taskKilledEvent = TaskKilledLifecycleEvent.builder()
- .taskExecutionRunnable(taskExecutionRunnable)
- .endTime(new Date(taskExecutionKilledEvent.getEndTime()))
- .build();
- taskExecutionRunnable.getWorkflowEventBus().publish(taskKilledEvent);
- // Once the master receive the event, then will send ack to the worker
- // This means the worker will not retry to send the event to the master
- // So once the master failover and we take over the task instance success, then we should fetch the latest task
- // instance state.
- if (!TaskTypeUtils.isLogicTask(taskExecutionRunnable.getTaskDefinition().getTaskType())) {
- Clients
- .withService(ITaskInstanceExecutionEventAckListener.class)
- .withHost(taskExecutionKilledEvent.getTaskInstanceHost())
- .handleTaskExecutionKilledEventAck(
- TaskExecutionKilledEventAck.success(taskExecutionKilledEvent.getTaskInstanceId()));
- }
- }
-
- @Override
- public void onTaskInstanceExecutionPaused(TaskExecutionPausedEvent taskExecutionPausedEvent) {
- final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutionPausedEvent);
- final TaskPausedLifecycleEvent taskPausedEvent = TaskPausedLifecycleEvent.of(taskExecutionRunnable);
- taskExecutionRunnable.getWorkflowEventBus().publish(taskPausedEvent);
- // Once the master receive the event, then will send ack to the worker
- // This means the worker will not retry to send the event to the master
- // So once the master failover and we take over the task instance success, then we should fetch the latest task
- // instance state.
- if (!TaskTypeUtils.isLogicTask(taskExecutionRunnable.getTaskDefinition().getTaskType())) {
- Clients
- .withService(ITaskInstanceExecutionEventAckListener.class)
- .withHost(taskExecutionPausedEvent.getTaskInstanceHost())
- .handleTaskExecutionPausedEventAck(
- TaskExecutionPausedEventAck.success(taskExecutionPausedEvent.getTaskInstanceId()));
- }
- }
-
- private ITaskExecutionRunnable getTaskExecutionRunnable(final ITaskExecutionEvent taskExecutionEvent) {
- final int workflowInstanceId = taskExecutionEvent.getWorkflowInstanceId();
- final int taskInstanceId = taskExecutionEvent.getTaskInstanceId();
-
- final IWorkflowExecutionRunnable workflowExecutionRunnable = IWorkflowRepository.get(workflowInstanceId);
- if (workflowExecutionRunnable == null) {
- throw new IllegalArgumentException("Cannot find the WorkflowExecuteRunnable: " + workflowInstanceId);
- }
- final ITaskExecutionRunnable taskExecutionRunnable = workflowExecutionRunnable.getWorkflowExecuteContext()
- .getWorkflowExecutionGraph()
- .getTaskExecutionRunnableById(taskInstanceId);
- if (taskExecutionRunnable == null) {
- throw new IllegalArgumentException("Cannot find the TaskExecuteRunnable: " + taskInstanceId);
- }
- return taskExecutionRunnable;
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java
new file mode 100644
index 0000000000..0998cde8b6
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskExecutorEventListenerImpl.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.rpc;
+
+import org.apache.dolphinscheduler.extract.master.ITaskExecutorEventListener;
+import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskDispatchedLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskFailedLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskKilledLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskPausedLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRunningLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskRuntimeContextChangedEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.lifecycle.event.TaskSuccessLifecycleEvent;
+import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
+import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
+import org.apache.dolphinscheduler.task.executor.events.IReportableTaskExecutorLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorDispatchedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorFailedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorKilledLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorPausedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorRuntimeContextChangedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorStartedLifecycleEvent;
+import org.apache.dolphinscheduler.task.executor.events.TaskExecutorSuccessLifecycleEvent;
+
+import java.util.Date;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+@Slf4j
+@Service
+public class TaskExecutorEventListenerImpl implements ITaskExecutorEventListener {
+
+ @Autowired
+ private IWorkflowRepository workflowRepository;
+
+ @Override
+ public void onTaskExecutorDispatched(final TaskExecutorDispatchedLifecycleEvent taskExecutorDispatchedLifecycleEvent) {
+ final ITaskExecutionRunnable taskExecutionRunnable =
+ getTaskExecutionRunnable(taskExecutorDispatchedLifecycleEvent);
+ final TaskDispatchedLifecycleEvent taskDispatchedLifecycleEvent = TaskDispatchedLifecycleEvent.builder()
+ .taskExecutionRunnable(taskExecutionRunnable)
+ .executorHost(taskExecutorDispatchedLifecycleEvent.getTaskInstanceHost())
+ .build();
+
+ taskExecutionRunnable.getWorkflowEventBus().publish(taskDispatchedLifecycleEvent);
+ }
+
+ @Override
+ public void onTaskExecutorRunning(final TaskExecutorStartedLifecycleEvent taskExecutorStartedLifecycleEvent) {
+ final ITaskExecutionRunnable taskExecutionRunnable =
+ getTaskExecutionRunnable(taskExecutorStartedLifecycleEvent);
+ final TaskRunningLifecycleEvent taskRunningEvent = TaskRunningLifecycleEvent.builder()
+ .taskExecutionRunnable(taskExecutionRunnable)
+ .startTime(new Date(taskExecutorStartedLifecycleEvent.getStartTime()))
+ .logPath(taskExecutorStartedLifecycleEvent.getLogPath())
+ .build();
+
+ taskExecutionRunnable.getWorkflowEventBus().publish(taskRunningEvent);
+ }
+
+ @Override
+ public void onTaskExecutorRuntimeContextChanged(final TaskExecutorRuntimeContextChangedLifecycleEvent taskExecutorRuntimeContextChangedLifecycleEventr) {
+ final ITaskExecutionRunnable taskExecutionRunnable =
+ getTaskExecutionRunnable(taskExecutorRuntimeContextChangedLifecycleEventr);
+
+ final TaskRuntimeContextChangedEvent taskRuntimeContextChangedEvent = TaskRuntimeContextChangedEvent.builder()
+ .taskExecutionRunnable(taskExecutionRunnable)
+ .runtimeContext(taskExecutorRuntimeContextChangedLifecycleEventr.getAppIds())
+ .build();
+
+ taskExecutionRunnable.getWorkflowEventBus().publish(taskRuntimeContextChangedEvent);
+ }
+
+ @Override
+ public void onTaskExecutorSuccess(final TaskExecutorSuccessLifecycleEvent taskExecutorSuccessLifecycleEvent) {
+ final ITaskExecutionRunnable taskExecutionRunnable =
+ getTaskExecutionRunnable(taskExecutorSuccessLifecycleEvent);
+ final TaskSuccessLifecycleEvent taskSuccessEvent = TaskSuccessLifecycleEvent.builder()
+ .taskExecutionRunnable(taskExecutionRunnable)
+ .endTime(new Date(taskExecutorSuccessLifecycleEvent.getEndTime()))
+ .varPool(taskExecutorSuccessLifecycleEvent.getVarPool())
+ .build();
+ taskExecutionRunnable.getWorkflowEventBus().publish(taskSuccessEvent);
+ }
+
+ @Override
+ public void onTaskExecutorFailed(final TaskExecutorFailedLifecycleEvent taskExecutorFailedLifecycleEvent) {
+ final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorFailedLifecycleEvent);
+ final TaskFailedLifecycleEvent taskFailedEvent = TaskFailedLifecycleEvent.builder()
+ .taskExecutionRunnable(taskExecutionRunnable)
+ .endTime(new Date(taskExecutorFailedLifecycleEvent.getEndTime()))
+ .build();
+ taskExecutionRunnable.getWorkflowEventBus().publish(taskFailedEvent);
+ }
+
+ @Override
+ public void onTaskExecutorKilled(final TaskExecutorKilledLifecycleEvent taskExecutorKilledLifecycleEvent) {
+ final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorKilledLifecycleEvent);
+ final TaskKilledLifecycleEvent taskKilledEvent = TaskKilledLifecycleEvent.builder()
+ .taskExecutionRunnable(taskExecutionRunnable)
+ .endTime(new Date(taskExecutorKilledLifecycleEvent.getEndTime()))
+ .build();
+ taskExecutionRunnable.getWorkflowEventBus().publish(taskKilledEvent);
+ }
+
+ @Override
+ public void onTaskExecutorPaused(final TaskExecutorPausedLifecycleEvent taskExecutorPausedLifecycleEvent) {
+ final ITaskExecutionRunnable taskExecutionRunnable = getTaskExecutionRunnable(taskExecutorPausedLifecycleEvent);
+ final TaskPausedLifecycleEvent taskPausedEvent = TaskPausedLifecycleEvent.of(taskExecutionRunnable);
+ taskExecutionRunnable.getWorkflowEventBus().publish(taskPausedEvent);
+ }
+
+ private ITaskExecutionRunnable getTaskExecutionRunnable(final IReportableTaskExecutorLifecycleEvent reportableTaskExecutorLifecycleEvent) {
+ final int workflowInstanceId = reportableTaskExecutorLifecycleEvent.getWorkflowInstanceId();
+ final int taskInstanceId = reportableTaskExecutorLifecycleEvent.getTaskInstanceId();
+
+ final IWorkflowExecutionRunnable workflowExecutionRunnable = workflowRepository.get(workflowInstanceId);
+ if (workflowExecutionRunnable == null) {
+ throw new IllegalArgumentException("Cannot find the WorkflowExecuteRunnable: " + workflowInstanceId);
+ }
+ final ITaskExecutionRunnable taskExecutionRunnable = workflowExecutionRunnable.getWorkflowExecuteContext()
+ .getWorkflowExecutionGraph()
+ .getTaskExecutionRunnableById(taskInstanceId);
+ if (taskExecutionRunnable == null) {
+ throw new IllegalArgumentException("Cannot find the TaskExecuteRunnable: " + taskInstanceId);
+ }
+ return taskExecutionRunnable;
+ }
+
+}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java
index e2657b697d..23883da8b6 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/GlobalTaskDispatchWaitingQueueLooper.java
@@ -20,9 +20,8 @@ package org.apache.dolphinscheduler.server.master.runner;
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
+import org.apache.dolphinscheduler.server.master.engine.task.client.ITaskExecutorClient;
import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.runner.dispatcher.TaskDispatchFactory;
-import org.apache.dolphinscheduler.server.master.runner.dispatcher.TaskDispatcher;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -39,7 +38,7 @@ public class GlobalTaskDispatchWaitingQueueLooper extends BaseDaemonThread imple
private GlobalTaskDispatchWaitingQueue globalTaskDispatchWaitingQueue;
@Autowired
- private TaskDispatchFactory taskDispatchFactory;
+ private ITaskExecutorClient taskExecutorClient;
private final AtomicBoolean RUNNING_FLAG = new AtomicBoolean(false);
@@ -74,8 +73,7 @@ public class GlobalTaskDispatchWaitingQueueLooper extends BaseDaemonThread imple
log.warn("The TaskInstance {} state is : {}, will not dispatch", taskInstance.getName(), status);
return;
}
- final TaskDispatcher taskDispatcher = taskDispatchFactory.getTaskDispatcher(taskInstance);
- taskDispatcher.dispatchTask(taskExecutionRunnable);
+ taskExecutorClient.dispatch(taskExecutionRunnable);
} catch (Exception e) {
// If dispatch failed, will put the task back to the queue
// The task will be dispatched after waiting time.
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecutorBootstrap.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecutorBootstrap.java
deleted file mode 100644
index 7d3bf6940b..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterTaskExecutorBootstrap.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner;
-
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncMasterTaskDelayQueueLooper;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class MasterTaskExecutorBootstrap implements AutoCloseable {
-
- @Autowired
- private GlobalTaskDispatchWaitingQueueLooper globalTaskDispatchWaitingQueueLooper;
-
- @Autowired
- private AsyncMasterTaskDelayQueueLooper asyncMasterTaskDelayQueueLooper;
-
- public synchronized void start() {
- log.info("MasterTaskExecutorBootstrap starting...");
- globalTaskDispatchWaitingQueueLooper.start();
- asyncMasterTaskDelayQueueLooper.start();
- log.info("MasterTaskExecutorBootstrap started...");
- }
-
- @Override
- public void close() throws Exception {
- log.info("MasterTaskExecutorBootstrap closing...");
- try (
- final GlobalTaskDispatchWaitingQueueLooper globalTaskDispatchWaitingQueueLooper1 =
- globalTaskDispatchWaitingQueueLooper;
- final AsyncMasterTaskDelayQueueLooper asyncMasterTaskDelayQueueLooper1 =
- asyncMasterTaskDelayQueueLooper) {
- // closed the resource
- }
- log.info("MasterTaskExecutorBootstrap closed...");
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecuteRunnableFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecuteRunnableFactory.java
deleted file mode 100644
index 4a97213c62..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecuteRunnableFactory.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner;
-
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.TaskExecuteRunnableCreateException;
-
-/**
- * Use to create TaskExecuteRunnable
- *
- * @param <T> TaskExecuteRunnable
- */
-public interface TaskExecuteRunnableFactory<T extends ITaskExecutionRunnable> {
-
- /**
- * Create a task execution runnable which related to the task code
- *
- * @param workflowExecuteRunnable
- * @param taskCode
- * @throws TaskExecuteRunnableCreateException
- */
- ITaskExecutionRunnable createTaskExecutionRunnable(IWorkflowExecutionRunnable workflowExecuteRunnable,
- Long taskCode);
-
- T createTaskExecuteRunnable(TaskInstance taskInstance) throws TaskExecuteRunnableCreateException;
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/BaseTaskDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/BaseTaskDispatcher.java
deleted file mode 100644
index 262b302d9b..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/BaseTaskDispatcher.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.dispatcher;
-
-import org.apache.dolphinscheduler.extract.base.utils.Host;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
-import org.apache.dolphinscheduler.server.master.exception.dispatch.WorkerGroupNotFoundException;
-
-import java.util.Optional;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public abstract class BaseTaskDispatcher implements TaskDispatcher {
-
- @Override
- public void dispatchTask(ITaskExecutionRunnable taskExecutionRunnable) throws TaskDispatchException {
- final TaskExecutionContext taskExecutionContext = taskExecutionRunnable.getTaskExecutionContext();
- final String taskName = taskExecutionRunnable.getTaskExecutionContext().getTaskName();
- final String taskInstanceDispatchAddress;
- try {
- taskInstanceDispatchAddress = getTaskInstanceDispatchHost(taskExecutionRunnable)
- .map(Host::getAddress)
- .orElseThrow(() -> new TaskDispatchException("Cannot find the host to execute task: " + taskName));
- } catch (WorkerGroupNotFoundException workerGroupNotFoundException) {
- // todo: this is a temporary solution, we should refactor the ServerNodeManager to make sure there won't
- // throw WorkerGroupNotFoundException unless the worker group is not exist in database
- throw new TaskDispatchException("Dispatch task: " + taskName + " failed", workerGroupNotFoundException);
- }
- // We inject the host here to avoid when we dispatched the task to worker, but the worker is crash.
- // Then we can use the host to do worker failover.
- taskExecutionContext.setHost(taskInstanceDispatchAddress);
- taskExecutionRunnable.getTaskInstance().setHost(taskInstanceDispatchAddress);
- doDispatch(taskExecutionRunnable);
- // todo: update the task state and host here, otherwise when the master failover the task host is null
- // but it already dispatched to worker
- // Or when the worker receive the task, it should wait the master send a start event to it.
- // the second solution is better
- log.info("Success dispatch task {} to {}.", taskName, taskInstanceDispatchAddress);
- }
-
- protected abstract void doDispatch(ITaskExecutionRunnable ITaskExecutionRunnable) throws TaskDispatchException;
-
- protected abstract Optional<Host> getTaskInstanceDispatchHost(ITaskExecutionRunnable taskExecutionContext) throws TaskDispatchException, WorkerGroupNotFoundException;
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcher.java
deleted file mode 100644
index ae87b6e33e..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/MasterTaskDispatcher.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.dispatcher;
-
-import org.apache.dolphinscheduler.extract.base.utils.Host;
-import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
-
-import java.util.Optional;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class MasterTaskDispatcher extends BaseTaskDispatcher {
-
- private final Optional<Host> masterTaskExecuteHost;
-
- @Autowired
- private ILogicTaskInstanceOperator logicTaskInstanceOperator;
-
- public MasterTaskDispatcher(MasterConfig masterConfig) {
- this.masterTaskExecuteHost = Optional.of(Host.of(masterConfig.getMasterAddress()));
- }
-
- @Override
- protected void doDispatch(final ITaskExecutionRunnable taskExecutionRunnable) throws TaskDispatchException {
- final TaskExecutionContext taskExecutionContext = taskExecutionRunnable.getTaskExecutionContext();
- try {
- final LogicTaskDispatchRequest logicTaskDispatchRequest =
- new LogicTaskDispatchRequest(taskExecutionContext);
- final LogicTaskDispatchResponse logicTaskDispatchResponse = logicTaskInstanceOperator.dispatchLogicTask(
- logicTaskDispatchRequest);
- if (!logicTaskDispatchResponse.isDispatchSuccess()) {
- throw new TaskDispatchException(
- String.format("Dispatch LogicTask to %s failed, response is: %s",
- taskExecutionContext.getHost(), logicTaskDispatchResponse));
- }
- } catch (TaskDispatchException e) {
- throw e;
- } catch (Exception e) {
- throw new TaskDispatchException(String.format("Dispatch task to %s failed",
- taskExecutionContext.getHost()), e);
- }
- }
-
- @Override
- protected Optional<Host> getTaskInstanceDispatchHost(ITaskExecutionRunnable taskExecutionContext) {
- return masterTaskExecuteHost;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatcher.java
deleted file mode 100644
index 4f612a61ab..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/TaskDispatcher.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.dispatcher;
-
-import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
-
-/**
- * Used to do task dispatcher.
- */
-public interface TaskDispatcher {
-
- void dispatchTask(ITaskExecutionRunnable ITaskExecutionRunnable) throws TaskDispatchException;
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcher.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcher.java
deleted file mode 100644
index 9ff7f75772..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/dispatcher/WorkerTaskDispatcher.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.dispatcher;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import org.apache.dolphinscheduler.extract.base.client.Clients;
-import org.apache.dolphinscheduler.extract.base.utils.Host;
-import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchRequest;
-import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceDispatchResponse;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.cluster.loadbalancer.IWorkerLoadBalancer;
-import org.apache.dolphinscheduler.server.master.engine.task.runnable.ITaskExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.dispatch.TaskDispatchException;
-
-import java.util.Optional;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class WorkerTaskDispatcher extends BaseTaskDispatcher {
-
- private final IWorkerLoadBalancer workerLoadBalancer;
-
- public WorkerTaskDispatcher(IWorkerLoadBalancer workerLoadBalancer) {
- this.workerLoadBalancer = checkNotNull(workerLoadBalancer);
- }
-
- @Override
- protected void doDispatch(ITaskExecutionRunnable ITaskExecutionRunnable) throws TaskDispatchException {
- final TaskExecutionContext taskExecutionContext = ITaskExecutionRunnable.getTaskExecutionContext();
- final String taskName = taskExecutionContext.getTaskName();
- final String workerAddress = taskExecutionContext.getHost();
- try {
- final TaskInstanceDispatchResponse taskInstanceDispatchResponse = Clients
- .withService(ITaskInstanceOperator.class)
- .withHost(workerAddress)
- .dispatchTask(new TaskInstanceDispatchRequest(ITaskExecutionRunnable.getTaskExecutionContext()));
- if (!taskInstanceDispatchResponse.isDispatchSuccess()) {
- throw new TaskDispatchException("Dispatch task: " + taskName + " to " + workerAddress + " failed: "
- + taskInstanceDispatchResponse);
- }
- } catch (TaskDispatchException e) {
- throw e;
- } catch (Exception e) {
- throw new TaskDispatchException("Dispatch task: " + taskName + " to " + workerAddress + " failed", e);
- }
- }
-
- @Override
- protected Optional<Host> getTaskInstanceDispatchHost(ITaskExecutionRunnable ITaskExecutionRunnable) {
- String workerGroup = ITaskExecutionRunnable.getTaskExecutionContext().getWorkerGroup();
- return workerLoadBalancer.select(workerGroup).map(Host::of);
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueueLooper.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueueLooper.java
deleted file mode 100644
index f797a684fb..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskDelayQueueLooper.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class AsyncMasterTaskDelayQueueLooper extends BaseDaemonThread implements AutoCloseable {
-
- @Autowired
- private AsyncMasterTaskDelayQueue asyncMasterTaskDelayQueue;
-
- @Autowired
- private MasterAsyncTaskExecutorThreadPool masterAsyncTaskExecutorThreadPool;
- private static final AtomicBoolean RUNNING_FLAG = new AtomicBoolean(false);
-
- public AsyncMasterTaskDelayQueueLooper() {
- super("AsyncMasterTaskDelayQueueLooper");
- }
-
- @Override
- public synchronized void start() {
- if (!RUNNING_FLAG.compareAndSet(false, true)) {
- log.info("The AsyncMasterTaskDelayQueueLooper has already been started, will not start again");
- return;
- }
-
- log.info("AsyncMasterTaskDelayQueueLooper starting...");
- super.start();
- log.info("AsyncMasterTaskDelayQueueLooper started...");
- }
-
- @Override
- public void run() {
- while (RUNNING_FLAG.get()) {
- AsyncTaskExecutionContext asyncTaskExecutionContext;
- try {
- asyncTaskExecutionContext = asyncMasterTaskDelayQueue.pollAsyncTask();
- } catch (InterruptedException e) {
- log.error("AsyncConditionTaskLooper has been interrupted, will break this loop", e);
- Thread.currentThread().interrupt();
- break;
- }
- final TaskExecutionContext taskExecutionContext = asyncTaskExecutionContext.getTaskExecutionContext();
- try {
- LogUtils.setWorkflowAndTaskInstanceIDMDC(taskExecutionContext.getWorkflowInstanceId(),
- taskExecutionContext.getTaskInstanceId());
- LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath());
-
- if (MasterTaskExecutionContextHolder
- .getTaskExecutionContext(taskExecutionContext.getTaskInstanceId()) == null) {
- log.warn(
- "Cannot find the taskInstance from TaskExecutionContextCacheManager, the task may already been killed, will stop the async master task");
- continue;
- }
- masterAsyncTaskExecutorThreadPool.getThreadPool().execute(() -> {
- final AsyncTaskExecuteFunction asyncTaskExecuteFunction =
- asyncTaskExecutionContext.getAsyncTaskExecuteFunction();
- final AsyncTaskCallbackFunction asyncTaskCallbackFunction =
- asyncTaskExecutionContext.getAsyncTaskCallbackFunction();
- try {
- LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath());
- LogUtils.setTaskInstanceIdMDC(taskExecutionContext.getTaskInstanceId());
- AsyncTaskExecuteFunction.AsyncTaskExecutionStatus asyncTaskExecutionStatus =
- asyncTaskExecuteFunction.getAsyncTaskExecutionStatus();
- switch (asyncTaskExecutionStatus) {
- case RUNNING:
- // If the task status is running, means the task real status is not finished. We
- // will
- // put it back to the queue to get the status again.
- asyncMasterTaskDelayQueue.addAsyncTask(asyncTaskExecutionContext);
- break;
- case PAUSE:
- asyncTaskCallbackFunction.executePause();
- break;
- case KILL:
- asyncTaskCallbackFunction.executeKilled();
- break;
- case SUCCESS:
- asyncTaskCallbackFunction.executeSuccess();
- break;
- case FAILED:
- asyncTaskCallbackFunction.executeFailed();
- break;
- }
- } catch (Exception ex) {
- asyncTaskCallbackFunction.executeThrowing(ex);
- } finally {
- LogUtils.removeTaskInstanceLogFullPathMDC();
- LogUtils.removeTaskInstanceIdMDC();
- }
- });
- } finally {
- LogUtils.removeTaskInstanceLogFullPathMDC();
- LogUtils.removeWorkflowAndTaskInstanceIdMDC();
- }
- }
- log.info("AsyncMasterTaskDelayQueueLooper closed...");
- }
-
- @Override
- public void close() throws Exception {
- if (!RUNNING_FLAG.compareAndSet(true, false)) {
- log.warn("The AsyncMasterTaskDelayQueueLooper is not started, will not close");
- return;
- }
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskExecutor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskExecutor.java
deleted file mode 100644
index d289a318da..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskExecutor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-import org.apache.dolphinscheduler.server.master.runner.task.IAsyncLogicTask;
-import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class AsyncMasterTaskExecutor extends MasterTaskExecutor {
-
- private final AsyncMasterTaskDelayQueue asyncMasterTaskDelayQueue;
-
- public AsyncMasterTaskExecutor(TaskExecutionContext taskExecutionContext,
- LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder,
- LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager,
- AsyncMasterTaskDelayQueue asyncTaskDelayQueue) {
- super(taskExecutionContext, logicTaskPluginFactoryBuilder, logicTaskInstanceExecutionEventSenderManager);
- this.asyncMasterTaskDelayQueue = asyncTaskDelayQueue;
- }
-
- @Override
- protected void executeTask() throws MasterTaskExecuteException {
- if (logicTask == null) {
- throw new MasterTaskExecuteException("The task plugin instance is null");
- }
- final IAsyncLogicTask iAsyncLogicTask = (IAsyncLogicTask) logicTask;
- // we execute the handle method here, but for async task, this method will not block
- // submit the task to async task queue
- final AsyncTaskExecutionContext asyncTaskExecutionContext = new AsyncTaskExecutionContext(
- taskExecutionContext,
- iAsyncLogicTask.getAsyncTaskExecuteFunction(),
- new AsyncTaskCallbackFunctionImpl(this));
- asyncMasterTaskDelayQueue.addAsyncTask(asyncTaskExecutionContext);
- }
-
- @Override
- protected void afterExecute() {
- // Do nothing, async task is not finished yet
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskExecutorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskExecutorFactory.java
deleted file mode 100644
index 6c81cd7862..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncMasterTaskExecutorFactory.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Component
-public class AsyncMasterTaskExecutorFactory
- implements
- MasterTaskExecutorFactory<AsyncMasterTaskExecutor> {
-
- @Autowired
- private LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder;
-
- @Autowired
- private LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager;
-
- @Autowired
- private AsyncMasterTaskDelayQueue asyncTaskDelayQueue;
-
- @Override
- public AsyncMasterTaskExecutor createMasterTaskExecutor(TaskExecutionContext taskExecutionContext) {
- return new AsyncMasterTaskExecutor(taskExecutionContext,
- logicTaskPluginFactoryBuilder,
- logicTaskInstanceExecutionEventSenderManager,
- asyncTaskDelayQueue);
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncTaskCallbackFunctionImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncTaskCallbackFunctionImpl.java
deleted file mode 100644
index ae7a480cd0..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncTaskCallbackFunctionImpl.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
-import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader;
-
-import lombok.NonNull;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class AsyncTaskCallbackFunctionImpl implements AsyncTaskCallbackFunction {
-
- private final AsyncMasterTaskExecutor asyncMasterTaskExecuteRunnable;
-
- public AsyncTaskCallbackFunctionImpl(@NonNull AsyncMasterTaskExecutor asyncMasterTaskExecuteRunnable) {
- this.asyncMasterTaskExecuteRunnable = asyncMasterTaskExecuteRunnable;
- }
-
- @Override
- public void executeSuccess() {
- asyncMasterTaskExecuteRunnable.getTaskExecutionContext()
- .setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
- executeFinished();
- }
-
- @Override
- public void executeFailed() {
- asyncMasterTaskExecuteRunnable.getTaskExecutionContext()
- .setCurrentExecutionStatus(TaskExecutionStatus.FAILURE);
- executeFinished();
- }
-
- @Override
- public void executeThrowing(Throwable throwable) {
- asyncMasterTaskExecuteRunnable.afterThrowing(throwable);
- }
-
- @Override
- public void executePause() {
- asyncMasterTaskExecuteRunnable.getTaskExecutionContext()
- .setCurrentExecutionStatus(TaskExecutionStatus.PAUSE);
- executeFinished();
- }
-
- @Override
- public void executeKilled() {
- asyncMasterTaskExecuteRunnable.getTaskExecutionContext()
- .setCurrentExecutionStatus(TaskExecutionStatus.KILL);
- executeFinished();
- }
-
- private void executeFinished() {
- TaskInstanceLogHeader.printFinalizeTaskHeader();
- int taskInstanceId = asyncMasterTaskExecuteRunnable.getTaskExecutionContext().getTaskInstanceId();
- MasterTaskExecutionContextHolder.removeTaskExecutionContext(taskInstanceId);
- MasterTaskExecutorHolder.removeMasterTaskExecutor(taskInstanceId);
- log.info("Task execute finished, removed the TaskExecutionContext");
- asyncMasterTaskExecuteRunnable.sendTaskResult();
- log.info(
- "Execute task finished, will send the task execute result to master, the current task execute result is {}",
- asyncMasterTaskExecuteRunnable.getTaskExecutionContext().getCurrentExecutionStatus().name());
- asyncMasterTaskExecuteRunnable.closeLogAppender();
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncTaskExecutionContext.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncTaskExecutionContext.java
deleted file mode 100644
index 14ec091886..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/AsyncTaskExecutionContext.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-
-import java.util.concurrent.Delayed;
-import java.util.concurrent.TimeUnit;
-
-import lombok.Data;
-import lombok.NonNull;
-
-@Data
-public class AsyncTaskExecutionContext implements Delayed {
-
- private final TaskExecutionContext taskExecutionContext;
-
- private final AsyncTaskExecuteFunction asyncTaskExecuteFunction;
-
- private final AsyncTaskCallbackFunction asyncTaskCallbackFunction;
-
- private long currentStartTime;
- private int executeTimes;
- private final long executeInterval;
-
- public AsyncTaskExecutionContext(@NonNull TaskExecutionContext taskExecutionContext,
- @NonNull AsyncTaskExecuteFunction asyncTaskExecuteFunction,
- @NonNull AsyncTaskCallbackFunction asyncTaskCallbackFunction) {
- this.taskExecutionContext = taskExecutionContext;
- this.asyncTaskExecuteFunction = asyncTaskExecuteFunction;
- this.asyncTaskCallbackFunction = asyncTaskCallbackFunction;
- this.currentStartTime = 0;
- this.executeTimes = 0;
- this.executeInterval = Math.max(asyncTaskExecuteFunction.getAsyncTaskStateCheckInterval().toMillis(), 1000L);
- }
-
- public synchronized void refreshStartTime() {
- if (executeTimes != 0) {
- // The first time doesn't have delay
- currentStartTime = System.currentTimeMillis();
- }
- executeTimes++;
- }
-
- @Override
- public long getDelay(TimeUnit unit) {
- long nextExecuteTimeDelay = currentStartTime + executeInterval - System.currentTimeMillis();
- return unit.convert(nextExecuteTimeDelay, TimeUnit.MILLISECONDS);
- }
-
- @Override
- public int compareTo(Delayed o) {
- if (o == null) {
- return 1;
- }
- return Long.compare(this.getDelay(TimeUnit.SECONDS), o.getDelay(TimeUnit.SECONDS));
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/IMasterTaskExecutorThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/IMasterTaskExecutorThreadPool.java
deleted file mode 100644
index 6c8e2caa73..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/IMasterTaskExecutorThreadPool.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-public interface IMasterTaskExecutorThreadPool<T extends MasterTaskExecutor> {
-
- boolean submitMasterTaskExecutor(T masterTaskExecutor);
-
- boolean removeMasterTaskExecutor(T masterTaskExecutor);
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterAsyncTaskExecutorThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterAsyncTaskExecutorThreadPool.java
deleted file mode 100644
index 2761958cd6..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterAsyncTaskExecutorThreadPool.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.common.thread.ThreadUtils;
-import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-
-import java.util.concurrent.ThreadPoolExecutor;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class MasterAsyncTaskExecutorThreadPool implements IMasterTaskExecutorThreadPool<AsyncMasterTaskExecutor> {
-
- private final ThreadPoolExecutor threadPoolExecutor;
-
- public MasterAsyncTaskExecutorThreadPool(MasterConfig masterConfig) {
- this.threadPoolExecutor = ThreadUtils.newDaemonFixedThreadExecutor("MasterAsyncTaskExecutorThreadPool",
- masterConfig.getMasterSyncTaskExecutorThreadPoolSize());
- }
-
- @Override
- public boolean submitMasterTaskExecutor(AsyncMasterTaskExecutor asyncMasterTaskExecutor) {
- synchronized (MasterAsyncTaskExecutorThreadPool.class) {
- // todo: check if the thread pool is overload
- threadPoolExecutor.execute(asyncMasterTaskExecutor);
- return true;
- }
- }
-
- @Override
- public boolean removeMasterTaskExecutor(AsyncMasterTaskExecutor asyncMasterTaskExecutor) {
- return threadPoolExecutor.remove(asyncMasterTaskExecutor);
- }
-
- // todo: remove this method, it's not a good idea to expose the ThreadPoolExecutor to out side.
- ThreadPoolExecutor getThreadPool() {
- return threadPoolExecutor;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterSyncTaskExecutorThreadPool.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterSyncTaskExecutorThreadPool.java
deleted file mode 100644
index 4ce59f1f49..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterSyncTaskExecutorThreadPool.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.common.thread.ThreadUtils;
-import org.apache.dolphinscheduler.server.master.config.MasterConfig;
-
-import java.util.concurrent.ThreadPoolExecutor;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class MasterSyncTaskExecutorThreadPool implements IMasterTaskExecutorThreadPool<SyncMasterTaskExecutor> {
-
- private final ThreadPoolExecutor threadPoolExecutor;
-
- public MasterSyncTaskExecutorThreadPool(MasterConfig masterConfig) {
- this.threadPoolExecutor = ThreadUtils.newDaemonFixedThreadExecutor("MasterSyncTaskExecutorThreadPool",
- masterConfig.getMasterSyncTaskExecutorThreadPoolSize());
- }
-
- @Override
- public boolean submitMasterTaskExecutor(SyncMasterTaskExecutor syncMasterTaskExecutor) {
- synchronized (MasterSyncTaskExecutorThreadPool.class) {
- // todo: check if the thread pool is overload
- threadPoolExecutor.execute(syncMasterTaskExecutor);
- return true;
- }
- }
-
- @Override
- public boolean removeMasterTaskExecutor(SyncMasterTaskExecutor syncMasterTaskExecutor) {
- return threadPoolExecutor.remove(syncMasterTaskExecutor);
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutionContextHolder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutionContextHolder.java
deleted file mode 100644
index edcbbe6708..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutionContextHolder.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class MasterTaskExecutionContextHolder {
-
- private static final Map<Integer, TaskExecutionContext> TASK_EXECUTION_CONTEXT_MAP = new ConcurrentHashMap<>();
-
- public static void putTaskExecutionContext(TaskExecutionContext taskExecutionContext) {
- if (TASK_EXECUTION_CONTEXT_MAP.containsKey(taskExecutionContext.getTaskInstanceId())) {
- log.error("The TaskExecutionContext {} already exists in the MasterTaskExecutionContextHolder",
- taskExecutionContext);
- }
- TASK_EXECUTION_CONTEXT_MAP.put(taskExecutionContext.getTaskInstanceId(), taskExecutionContext);
- }
-
- public static TaskExecutionContext getTaskExecutionContext(int taskInstanceId) {
- return TASK_EXECUTION_CONTEXT_MAP.get(taskInstanceId);
- }
-
- public static void removeTaskExecutionContext(int taskInstanceId) {
- TASK_EXECUTION_CONTEXT_MAP.remove(taskInstanceId);
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutor.java
deleted file mode 100644
index d21e81fe97..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutor.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER;
-import static org.apache.dolphinscheduler.common.constants.Constants.DRY_RUN_FLAG_YES;
-
-import org.apache.dolphinscheduler.common.log.remote.RemoteLogUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
-import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader;
-import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
-import org.apache.dolphinscheduler.server.master.exception.LogicTaskFactoryNotFoundException;
-import org.apache.dolphinscheduler.server.master.exception.LogicTaskInitializeException;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-import org.apache.dolphinscheduler.server.master.runner.task.ILogicTask;
-import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder;
-
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public abstract class MasterTaskExecutor implements Runnable {
-
- @Getter
- protected final TaskExecutionContext taskExecutionContext;
- @Getter
- protected ILogicTask logicTask;
-
- protected final LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder;
- protected final LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager;
-
- public MasterTaskExecutor(TaskExecutionContext taskExecutionContext,
- LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder,
- LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager) {
- this.taskExecutionContext = taskExecutionContext;
- this.logicTaskPluginFactoryBuilder = logicTaskPluginFactoryBuilder;
- this.logicTaskInstanceExecutionEventSenderManager = logicTaskInstanceExecutionEventSenderManager;
- }
-
- protected abstract void executeTask() throws MasterTaskExecuteException;
-
- protected abstract void afterExecute() throws MasterTaskExecuteException;
-
- protected void afterThrowing(Throwable throwable) {
- TaskInstanceLogHeader.printFinalizeTaskHeader();
- try {
- log.error("Get a exception when execute the task, will try to cancel the task", throwable);
- cancelTask();
- } catch (Exception e) {
- log.error("Cancel task failed,", e);
- }
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.FAILURE);
- sendTaskResult();
- log.info(
- "Get a exception when execute the task, sent the task execute result to master, the current task execute result is {}",
- taskExecutionContext.getCurrentExecutionStatus());
- MasterTaskExecutionContextHolder.removeTaskExecutionContext(taskExecutionContext.getTaskInstanceId());
- MasterTaskExecutorHolder.removeMasterTaskExecutor(taskExecutionContext.getTaskInstanceId());
- log.info("Get a exception when execute the task, removed the TaskExecutionContext");
- closeLogAppender();
- }
-
- public void cancelTask() throws MasterTaskExecuteException {
- if (logicTask != null) {
- logicTask.kill();
- }
- }
-
- public void pauseTask() throws MasterTaskExecuteException {
- if (logicTask != null) {
- logicTask.pause();
- }
- }
-
- @Override
- public void run() {
- try {
- LogUtils.setWorkflowAndTaskInstanceIDMDC(
- taskExecutionContext.getWorkflowInstanceId(),
- taskExecutionContext.getTaskInstanceId());
- LogUtils.setTaskInstanceLogFullPathMDC(taskExecutionContext.getLogPath());
-
- TaskInstanceLogHeader.printInitializeTaskContextHeader();
- initializeTask();
-
- TaskInstanceLogHeader.printLoadTaskInstancePluginHeader();
- beforeExecute();
-
- if (DRY_RUN_FLAG_YES == taskExecutionContext.getDryRun()) {
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.SUCCESS);
- taskExecutionContext.setEndTime(System.currentTimeMillis());
- MasterTaskExecutorHolder.removeMasterTaskExecutor(taskExecutionContext.getTaskInstanceId());
- logicTaskInstanceExecutionEventSenderManager.successEventSender().sendMessage(taskExecutionContext);
- log.info(
- "The current execute mode is dry run, will stop the logic task and set the taskInstance status to success");
- return;
- }
- TaskInstanceLogHeader.printExecuteTaskHeader();
- executeTask();
-
- afterExecute();
- } catch (Throwable ex) {
- log.error("Task execute failed, due to meet an exception", ex);
- afterThrowing(ex);
- } finally {
- LogUtils.removeWorkflowAndTaskInstanceIdMDC();
- LogUtils.removeTaskInstanceLogFullPathMDC();
- }
- }
-
- protected void initializeTask() {
- log.info("Begin to initialize task");
-
- long taskStartTime = System.currentTimeMillis();
- taskExecutionContext.setStartTime(taskStartTime);
- log.info("Set task startTime: {}", taskStartTime);
-
- String taskAppId = String.format("%s_%s", taskExecutionContext.getWorkflowInstanceId(),
- taskExecutionContext.getTaskInstanceId());
- taskExecutionContext.setTaskAppId(taskAppId);
- log.info("Set task appId: {}", taskAppId);
-
- log.info("End initialize task {}", JSONUtils.toPrettyJsonString(taskExecutionContext));
- }
-
- protected void beforeExecute() throws LogicTaskFactoryNotFoundException, LogicTaskInitializeException {
- taskExecutionContext.setCurrentExecutionStatus(TaskExecutionStatus.RUNNING_EXECUTION);
- logicTaskInstanceExecutionEventSenderManager.runningEventSender().sendMessage(taskExecutionContext);
- log.info("Send task status {} to master {}", taskExecutionContext.getCurrentExecutionStatus().name(),
- taskExecutionContext.getWorkflowInstanceHost());
-
- logicTask = logicTaskPluginFactoryBuilder.createILogicTaskPluginFactory(taskExecutionContext.getTaskType())
- .createLogicTask(taskExecutionContext);
- log.info("Initialized task plugin instance: {} successfully", taskExecutionContext.getTaskType());
-
- logicTask.getTaskParameters().setVarPool(taskExecutionContext.getVarPool());
- log.info("Initialize taskVarPool: {} successfully", taskExecutionContext.getVarPool());
-
- }
-
- protected void closeLogAppender() {
- try {
- if (RemoteLogUtils.isRemoteLoggingEnable()) {
- RemoteLogUtils.sendRemoteLog(taskExecutionContext.getLogPath());
- log.info("Send task log {} to remote storage successfully", taskExecutionContext.getLogPath());
- }
- } catch (Exception ex) {
- log.error("Send task log {} to remote storage failed", taskExecutionContext.getLogPath(), ex);
- } finally {
- log.info(FINALIZE_SESSION_MARKER, FINALIZE_SESSION_MARKER.toString());
- }
- }
-
- protected void sendTaskResult() {
- try {
- taskExecutionContext.setEndTime(System.currentTimeMillis());
- taskExecutionContext.setVarPool(JSONUtils.toJsonString(logicTask.getTaskParameters().getVarPool()));
- switch (taskExecutionContext.getCurrentExecutionStatus()) {
- case KILL:
- logicTaskInstanceExecutionEventSenderManager.killedEventSender().sendMessage(taskExecutionContext);
- break;
- case PAUSE:
- logicTaskInstanceExecutionEventSenderManager.pausedEventSender().sendMessage(taskExecutionContext);
- break;
- case FAILURE:
- logicTaskInstanceExecutionEventSenderManager.failedEventSender().sendMessage(taskExecutionContext);
- break;
- case SUCCESS:
- logicTaskInstanceExecutionEventSenderManager.successEventSender().sendMessage(taskExecutionContext);
- break;
- default:
- logicTaskInstanceExecutionEventSenderManager.failedEventSender().sendMessage(taskExecutionContext);
- break;
- }
- log.info("Send task status: {} to master: {} successfully",
- taskExecutionContext.getCurrentExecutionStatus().name(),
- taskExecutionContext.getWorkflowInstanceHost());
- } catch (Exception ex) {
- log.error("Send task status: {} to master: {} failed",
- taskExecutionContext.getCurrentExecutionStatus().name(),
- taskExecutionContext.getWorkflowInstanceHost(), ex);
- }
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorFactoryBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorFactoryBuilder.java
deleted file mode 100644
index 8fa6002463..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorFactoryBuilder.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.task.SubWorkflowLogicTaskChannelFactory;
-import org.apache.dolphinscheduler.server.master.runner.task.dependent.DependentLogicTask;
-import org.apache.dolphinscheduler.server.master.runner.task.dynamic.DynamicLogicTask;
-
-import java.util.Set;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import com.google.common.collect.Sets;
-
-/**
- * Use to create MasterTaskExecutorFactory
- */
-@Component
-public class MasterTaskExecutorFactoryBuilder {
-
- @Autowired
- private AsyncMasterTaskExecutorFactory asyncMasterTaskExecutorFactory;
-
- @Autowired
- private SyncMasterTaskExecutorFactory syncMasterTaskExecutorFactory;
-
- private static final Set<String> ASYNC_TASK_TYPE = Sets.newHashSet(
- DependentLogicTask.TASK_TYPE,
- SubWorkflowLogicTaskChannelFactory.NAME,
- DynamicLogicTask.TASK_TYPE);
-
- public MasterTaskExecutorFactory<? extends MasterTaskExecutor> createMasterTaskExecutorFactory(String taskType) {
- if (ASYNC_TASK_TYPE.contains(taskType)) {
- return asyncMasterTaskExecutorFactory;
- }
- return syncMasterTaskExecutorFactory;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorHolder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorHolder.java
deleted file mode 100644
index 962bfc6ed3..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorHolder.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import lombok.experimental.UtilityClass;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@UtilityClass
-public class MasterTaskExecutorHolder {
-
- private static final Map<Integer, MasterTaskExecutor> SUBMITTED_MASTER_TASK_MAP = new ConcurrentHashMap<>();
-
- public void putMasterTaskExecuteRunnable(MasterTaskExecutor masterTaskExecutor) {
- SUBMITTED_MASTER_TASK_MAP.put(masterTaskExecutor.getTaskExecutionContext().getTaskInstanceId(),
- masterTaskExecutor);
- }
-
- public MasterTaskExecutor getMasterTaskExecutor(Integer taskInstanceId) {
- return SUBMITTED_MASTER_TASK_MAP.get(taskInstanceId);
- }
-
- public void removeMasterTaskExecutor(Integer taskInstanceId) {
- SUBMITTED_MASTER_TASK_MAP.remove(taskInstanceId);
- }
-
- public Collection<MasterTaskExecutor> getAllMasterTaskExecutor() {
- return SUBMITTED_MASTER_TASK_MAP.values();
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorThreadPoolManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorThreadPoolManager.java
deleted file mode 100644
index 9c30d49f8f..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/MasterTaskExecutorThreadPoolManager.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class MasterTaskExecutorThreadPoolManager {
-
- @Autowired
- private MasterSyncTaskExecutorThreadPool masterSyncTaskExecutorThreadPool;
-
- @Autowired
- private MasterAsyncTaskExecutorThreadPool masterAsyncTaskExecutorThreadPool;
-
- @Autowired
- private LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager;
-
- public boolean submitMasterTaskExecutor(final MasterTaskExecutor masterTaskExecutor) {
- MasterTaskExecutorHolder.putMasterTaskExecuteRunnable(masterTaskExecutor);
- sendDispatchedEvent(masterTaskExecutor);
- if (masterTaskExecutor instanceof SyncMasterTaskExecutor) {
- return masterSyncTaskExecutorThreadPool
- .submitMasterTaskExecutor((SyncMasterTaskExecutor) masterTaskExecutor);
- }
- if (masterTaskExecutor instanceof AsyncMasterTaskExecutor) {
- return masterAsyncTaskExecutorThreadPool
- .submitMasterTaskExecutor((AsyncMasterTaskExecutor) masterTaskExecutor);
- }
- throw new IllegalArgumentException("Unknown type of MasterTaskExecutor: " + masterTaskExecutor);
- }
-
- public boolean removeMasterTaskExecutor(final MasterTaskExecutor masterTaskExecutor) {
- if (masterTaskExecutor instanceof SyncMasterTaskExecutor) {
- return masterSyncTaskExecutorThreadPool
- .removeMasterTaskExecutor((SyncMasterTaskExecutor) masterTaskExecutor);
- }
- if (masterTaskExecutor instanceof AsyncMasterTaskExecutor) {
- return masterAsyncTaskExecutorThreadPool
- .removeMasterTaskExecutor((AsyncMasterTaskExecutor) masterTaskExecutor);
- }
- throw new IllegalArgumentException("Unknown type of MasterTaskExecutor: " + masterTaskExecutor);
- }
-
- private void sendDispatchedEvent(final MasterTaskExecutor masterTaskExecutor) {
- logicTaskInstanceExecutionEventSenderManager.dispatchEventSender()
- .sendMessage(masterTaskExecutor.getTaskExecutionContext());
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterTaskExecutor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterTaskExecutor.java
deleted file mode 100644
index 7f303487ef..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterTaskExecutor.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-import org.apache.dolphinscheduler.server.master.runner.task.ISyncLogicTask;
-import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class SyncMasterTaskExecutor extends MasterTaskExecutor {
-
- public SyncMasterTaskExecutor(TaskExecutionContext taskExecutionContext,
- LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder,
- LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager) {
- super(taskExecutionContext, logicTaskPluginFactoryBuilder, logicTaskInstanceExecutionEventSenderManager);
- }
-
- @Override
- protected void executeTask() throws MasterTaskExecuteException {
- if (logicTask == null) {
- throw new MasterTaskExecuteException("The task plugin instance is null");
- }
- ISyncLogicTask iSyncLogicTask = (ISyncLogicTask) logicTask;
- iSyncLogicTask.handle();
- }
-
- protected void afterExecute() throws MasterTaskExecuteException {
- TaskInstanceLogHeader.printFinalizeTaskHeader();
- // todo: move this to constructor to avoid check every time
- if (logicTask == null) {
- throw new MasterTaskExecuteException("The current task instance is null");
- }
- sendTaskResult();
- log.info(
- "Execute task finished, will send the task execute result to master, the current task execute result is {}",
- taskExecutionContext.getCurrentExecutionStatus().name());
- closeLogAppender();
- MasterTaskExecutionContextHolder.removeTaskExecutionContext(taskExecutionContext.getTaskInstanceId());
- MasterTaskExecutorHolder.removeMasterTaskExecutor(taskExecutionContext.getTaskInstanceId());
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterTaskExecutorFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterTaskExecutorFactory.java
deleted file mode 100644
index b591711e3d..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/execute/SyncMasterTaskExecutorFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.execute;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.server.master.runner.message.LogicTaskInstanceExecutionEventSenderManager;
-import org.apache.dolphinscheduler.server.master.runner.task.LogicTaskPluginFactoryBuilder;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class SyncMasterTaskExecutorFactory
- implements
- MasterTaskExecutorFactory<SyncMasterTaskExecutor> {
-
- @Autowired
- private LogicTaskPluginFactoryBuilder logicTaskPluginFactoryBuilder;
- @Autowired
- private LogicTaskInstanceExecutionEventSenderManager logicTaskInstanceExecutionEventSenderManager;
-
- @Override
- public SyncMasterTaskExecutor createMasterTaskExecutor(TaskExecutionContext taskExecutionContext) {
- return new SyncMasterTaskExecutor(taskExecutionContext, logicTaskPluginFactoryBuilder,
- logicTaskInstanceExecutionEventSenderManager);
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteDispatchEventSender.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteDispatchEventSender.java
deleted file mode 100644
index 2f45baa51b..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteDispatchEventSender.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.message;
-
-import org.apache.dolphinscheduler.extract.base.client.Clients;
-import org.apache.dolphinscheduler.extract.master.ITaskExecutionEventListener;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionDispatchEvent;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-
-import org.springframework.stereotype.Component;
-
-@Component
-public class LogicTaskInstanceExecuteDispatchEventSender
- implements
- LogicTaskInstanceExecutionEventSender<TaskExecutionDispatchEvent> {
-
- @Override
- public void sendMessage(TaskExecutionDispatchEvent taskExecutionDispatchEvent) {
- Clients
- .withService(ITaskExecutionEventListener.class)
- .withHost(taskExecutionDispatchEvent.getWorkflowInstanceHost())
- .onTaskInstanceDispatched(taskExecutionDispatchEvent);
- }
-
- @Override
- public TaskExecutionDispatchEvent buildMessage(TaskExecutionContext taskExecutionContext) {
- final TaskExecutionDispatchEvent taskExecutionDispatchEvent =
- TaskExecutionDispatchEvent.builder()
- .workflowInstanceId(taskExecutionContext.getWorkflowInstanceId())
- .workflowInstanceHost(taskExecutionContext.getWorkflowInstanceHost())
- .taskInstanceId(taskExecutionContext.getTaskInstanceId())
- .taskInstanceHost(taskExecutionContext.getHost())
- .eventCreateTime(System.currentTimeMillis())
- .eventSendTime(System.currentTimeMillis())
- .build();
- return taskExecutionDispatchEvent;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteRunningEventSender.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteRunningEventSender.java
deleted file mode 100644
index 8e27246a91..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecuteRunningEventSender.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.message;
-
-import org.apache.dolphinscheduler.extract.base.client.Clients;
-import org.apache.dolphinscheduler.extract.master.ITaskExecutionEventListener;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskExecutionRunningEvent;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-
-import lombok.NonNull;
-
-import org.springframework.stereotype.Component;
-
-@Component
-public class LogicTaskInstanceExecuteRunningEventSender
- implements
- LogicTaskInstanceExecutionEventSender<TaskExecutionRunningEvent> {
-
- @Override
- public void sendMessage(TaskExecutionRunningEvent taskInstanceExecutionRunningEvent) {
- Clients
- .withService(ITaskExecutionEventListener.class)
- .withHost(taskInstanceExecutionRunningEvent.getWorkflowInstanceHost())
- .onTaskInstanceExecutionRunning(taskInstanceExecutionRunningEvent);
- }
-
- @Override
- public TaskExecutionRunningEvent buildMessage(@NonNull TaskExecutionContext taskExecutionContext) {
- TaskExecutionRunningEvent taskExecuteRunningMessage = new TaskExecutionRunningEvent();
- taskExecuteRunningMessage.setTaskInstanceId(taskExecutionContext.getTaskInstanceId());
- taskExecuteRunningMessage.setWorkflowInstanceId(taskExecutionContext.getWorkflowInstanceId());
- taskExecuteRunningMessage.setStatus(taskExecutionContext.getCurrentExecutionStatus());
- taskExecuteRunningMessage.setLogPath(taskExecutionContext.getLogPath());
- taskExecuteRunningMessage.setWorkflowInstanceHost(taskExecutionContext.getWorkflowInstanceHost());
- taskExecuteRunningMessage.setTaskInstanceHost(taskExecutionContext.getHost());
- taskExecuteRunningMessage.setStartTime(taskExecutionContext.getStartTime());
- taskExecuteRunningMessage.setExecutePath(taskExecutionContext.getExecutePath());
- taskExecuteRunningMessage.setAppIds(taskExecutionContext.getAppIds());
- return taskExecuteRunningMessage;
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSender.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSender.java
deleted file mode 100644
index 3b3ff7f645..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSender.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.message;
-
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-
-// todo: unify with WorkerMessageSender
-public interface LogicTaskInstanceExecutionEventSender<T> {
-
- /**
- * Send the message
- */
- void sendMessage(T message);
-
- default void sendMessage(TaskExecutionContext taskExecutionContext) {
- T message = buildMessage(taskExecutionContext);
- sendMessage(message);
- }
-
- /**
- * Build the message from task context and message received address.
- */
- T buildMessage(TaskExecutionContext taskExecutionContext);
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSenderManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSenderManager.java
deleted file mode 100644
index eb3ac11e13..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/message/LogicTaskInstanceExecutionEventSenderManager.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.message;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class LogicTaskInstanceExecutionEventSenderManager {
-
- @Autowired
- private LogicTaskInstanceExecuteDispatchEventSender logicTaskInstanceExecuteDispatchEventSender;
-
- @Autowired
- private LogicTaskInstanceExecuteRunningEventSender logicTaskInstanceExecuteRunningEventSender;
-
- @Autowired
- private LogicTaskInstanceExecutionFailedEventSender logicTaskInstanceExecutionFailedEventSender;
-
- @Autowired
- private LogicTaskInstanceExecutionPausedEventSender logicTaskInstanceExecutionPausedEventSender;
-
- @Autowired
- private LogicTaskInstanceExecutionSuccessEventSender logicTaskInstanceExecutionSuccessEventSender;
-
- @Autowired
- private LogicTaskInstanceExecutionKilledEventSender logicTaskInstanceExecutionKilledEventSender;
-
- public LogicTaskInstanceExecuteDispatchEventSender dispatchEventSender() {
- return logicTaskInstanceExecuteDispatchEventSender;
- }
-
- public LogicTaskInstanceExecuteRunningEventSender runningEventSender() {
- return logicTaskInstanceExecuteRunningEventSender;
- }
-
- public LogicTaskInstanceExecutionFailedEventSender failedEventSender() {
- return logicTaskInstanceExecutionFailedEventSender;
- }
-
- public LogicTaskInstanceExecutionPausedEventSender pausedEventSender() {
- return logicTaskInstanceExecutionPausedEventSender;
- }
-
- public LogicTaskInstanceExecutionSuccessEventSender successEventSender() {
- return logicTaskInstanceExecutionSuccessEventSender;
- }
-
- public LogicTaskInstanceExecutionKilledEventSender killedEventSender() {
- return logicTaskInstanceExecutionKilledEventSender;
- }
-
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseAsyncLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseAsyncLogicTask.java
deleted file mode 100644
index 2b9d5728a1..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseAsyncLogicTask.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.task;
-
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public abstract class BaseAsyncLogicTask<T extends AbstractParameters> implements IAsyncLogicTask {
-
- protected final TaskExecutionContext taskExecutionContext;
- protected final T taskParameters;
-
- protected BaseAsyncLogicTask(TaskExecutionContext taskExecutionContext, T taskParameters) {
- this.taskExecutionContext = taskExecutionContext;
- this.taskParameters = taskParameters;
- log.info("Success initialize task parameters: \n{}", JSONUtils.toPrettyJsonString(taskParameters));
- }
-
- @Override
- public TaskExecutionContext getTaskExecutionContext() {
- return taskExecutionContext;
- }
-
- @Override
- public AbstractParameters getTaskParameters() {
- return taskParameters;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java
deleted file mode 100644
index c971245b8c..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseSyncLogicTask.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.task;
-
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
-import org.apache.dolphinscheduler.server.master.engine.workflow.runnable.IWorkflowExecutionRunnable;
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public abstract class BaseSyncLogicTask<T extends AbstractParameters> implements ISyncLogicTask {
-
- protected final TaskExecutionContext taskExecutionContext;
-
- protected final IWorkflowExecutionRunnable workflowExecutionRunnable;
- protected final TaskInstance taskInstance;
- protected final T taskParameters;
-
- protected BaseSyncLogicTask(IWorkflowExecutionRunnable workflowExecutionRunnable,
- TaskExecutionContext taskExecutionContext,
- T taskParameters) {
- this.taskExecutionContext = taskExecutionContext;
- this.workflowExecutionRunnable = workflowExecutionRunnable;
- this.taskInstance = workflowExecutionRunnable
- .getWorkflowExecuteContext()
- .getWorkflowExecutionGraph()
- .getTaskExecutionRunnableById(taskExecutionContext.getTaskInstanceId())
- .getTaskInstance();
- this.taskParameters = taskParameters;
- log.info("Success initialize task parameters: \n{}", JSONUtils.toPrettyJsonString(taskParameters));
- }
-
- @Override
- public void pause() throws MasterTaskExecuteException {
- log.warn("The Sync task does not support pause operation");
- }
-
- @Override
- public void kill() throws MasterTaskExecuteException {
- log.warn("The Sync task does not support kill operation");
- }
-
- @Override
- public TaskExecutionContext getTaskExecutionContext() {
- return taskExecutionContext;
- }
-
- @Override
- public AbstractParameters getTaskParameters() {
- return taskParameters;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/IAsyncLogicTask.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/IAsyncLogicTask.java
deleted file mode 100644
index 88d06ed1e4..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/IAsyncLogicTask.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.runner.task;
-
-import org.apache.dolphinscheduler.server.master.exception.MasterTaskExecuteException;
-import org.apache.dolphinscheduler.server.master.runner.execute.AsyncTaskExecuteFunction;
-
-public interface IAsyncLogicTask extends ILogicTask {
-
- AsyncTaskExecuteFunction getAsyncTaskExecuteFunction() throws MasterTaskExecuteException;
-}
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index 5521105f44..1233d99acc 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -86,10 +86,9 @@ master:
listen-port: 5678
# The number of workflow event bus fire worker used to fire the workflow event, default is 2*CPU core + 1.
# workflow-event-bus-fire-thread-count: 10
- # The number of threads used to execute sync logic task e.g. Switch/Condition, default is CPU core.
- # master-sync-task-executor-thread-pool-size: 10
- # The number of threads used to execute async logic task e.g. Dependent/SubWorkflow default is CPU core.
- # master-async-task-executor-thread-pool-size: 10
+ # logic-task-config:
+ # The number of threads used to execute logic task.
+ # task-executor-thread-count: 10
max-heartbeat-interval: 10s
server-load-protection:
# If set true, will open master overload protection
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/AbstractMasterIntegrationTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/AbstractMasterIntegrationTestCase.java
index 764150f919..91d11ee714 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/AbstractMasterIntegrationTestCase.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/AbstractMasterIntegrationTestCase.java
@@ -18,9 +18,14 @@
package org.apache.dolphinscheduler.server.master;
import org.apache.dolphinscheduler.dao.DaoConfiguration;
+import org.apache.dolphinscheduler.server.master.integration.MasterContainer;
+import org.apache.dolphinscheduler.server.master.integration.Repository;
+import org.apache.dolphinscheduler.server.master.integration.WorkflowOperator;
+import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContextFactory;
import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.DirtiesContext;
@@ -36,4 +41,15 @@ import org.springframework.test.annotation.DirtiesContext;
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
public abstract class AbstractMasterIntegrationTestCase {
+ @Autowired
+ protected WorkflowTestCaseContextFactory workflowTestCaseContextFactory;
+
+ @Autowired
+ protected WorkflowOperator workflowOperator;
+
+ @Autowired
+ protected Repository repository;
+
+ @Autowired
+ protected MasterContainer masterContainer;
}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/MasterContainer.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/MasterContainer.java
new file mode 100644
index 0000000000..bbe3af3f12
--- /dev/null
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/MasterContainer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.integration;
+
+import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import org.apache.dolphinscheduler.eventbus.AbstractDelayEventBus;
+import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
+import org.apache.dolphinscheduler.server.master.engine.WorkflowEventBusFireWorkers;
+import org.apache.dolphinscheduler.server.master.engine.executor.LogicTaskExecutorContainerProvider;
+import org.apache.dolphinscheduler.server.master.engine.executor.LogicTaskExecutorLifecycleEventReporter;
+import org.apache.dolphinscheduler.server.master.engine.executor.LogicTaskExecutorRepository;
+import org.apache.dolphinscheduler.server.master.engine.system.SystemEventBus;
+import org.apache.dolphinscheduler.task.executor.container.AbstractTaskExecutorContainer;
+import org.apache.dolphinscheduler.task.executor.container.TaskExecutorAssignmentTable;
+import org.apache.dolphinscheduler.task.executor.worker.TaskExecutorWorkers;
+
+import java.util.concurrent.TimeUnit;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MasterContainer {
+
+ @Autowired
+ private SystemEventBus systemEventBus;
+
+ @Autowired
+ private IWorkflowRepository workflowRepository;
+
+ @Autowired
+ private WorkflowEventBusFireWorkers workflowEventBusFireWorkers;
+
+ @Autowired
+ private LogicTaskExecutorRepository logicTaskExecutorRepository;
+
+ @Autowired
+ private LogicTaskExecutorContainerProvider logicTaskExecutorContainerProvider;
+
+ @Autowired
+ private LogicTaskExecutorLifecycleEventReporter logicTaskExecutorLifecycleEventReporter;
+
+ public void assertAllResourceReleased() {
+ await()
+ .atMost(10, TimeUnit.SECONDS)
+ .untilAsserted(this::doAssertAllResourceReleased);
+ }
+
+ private void doAssertAllResourceReleased() {
+ assertThat(workflowRepository.getAll()).isEmpty();
+
+ assertThat(workflowEventBusFireWorkers.getWorkers())
+ .allMatch(workflowEventBusFireWorker -> workflowEventBusFireWorker
+ .getRegisteredWorkflowExecuteRunnableSize() == 0);
+ assertThat(systemEventBus).matches(AbstractDelayEventBus::isEmpty);
+
+ assertThat(logicTaskExecutorRepository.getAll()).isEmpty();
+
+ final AbstractTaskExecutorContainer executorContainer =
+ (AbstractTaskExecutorContainer) logicTaskExecutorContainerProvider.getExecutorContainer();
+ assertThat(executorContainer.getTaskExecutorAssignmentTable()).matches(TaskExecutorAssignmentTable::isEmpty);
+
+ final TaskExecutorWorkers taskExecutorWorkers = executorContainer.getTaskExecutorWorkers();
+ assertThat(taskExecutorWorkers.getWorkers())
+ .allMatch(taskExecutorWorker -> taskExecutorWorker.getRegisteredTaskExecutorSize() == 0)
+ .allMatch(taskExecutorWorker -> taskExecutorWorker.getFiredTaskExecutorSize() == 0);
+
+ assertThat(logicTaskExecutorLifecycleEventReporter.getEventChannels()).isEmpty();
+ }
+}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowOperator.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowOperator.java
index 3c30fe947c..c133e29a0d 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowOperator.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowOperator.java
@@ -95,6 +95,12 @@ public class WorkflowOperator {
schedulerApi.insertOrUpdateScheduleTask(project.getId(), schedule);
}
+ public void unSchedulingWorkflow(final WorkflowSchedulingDTO workflowSchedulingDTO) {
+ final Project project = workflowSchedulingDTO.getProject();
+ final Schedule schedule = workflowSchedulingDTO.getSchedule();
+ schedulerApi.deleteScheduleTask(project.getId(), schedule.getId());
+ }
+
public void repeatRunningWorkflowInstance(final Integer workflowInstanceId) {
final WorkflowInstanceRepeatRunningRequest repeatRunningRequest = WorkflowInstanceRepeatRunningRequest.builder()
.workflowInstanceId(workflowInstanceId)
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java
index c67ed9ec80..2c5853f33b 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/WorkflowTestCaseContext.java
@@ -24,6 +24,8 @@ import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition;
import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
import org.apache.dolphinscheduler.dao.entity.WorkflowTaskRelation;
+import org.apache.commons.collections4.CollectionUtils;
+
import java.util.List;
import lombok.AllArgsConstructor;
@@ -47,4 +49,11 @@ public class WorkflowTestCaseContext {
private List<WorkflowTaskRelation> taskRelations;
+ public WorkflowDefinition getOneWorkflow() {
+ if (CollectionUtils.isEmpty(workflows)) {
+ throw new IllegalStateException("workflows is empty");
+ }
+ return workflows.get(0);
+ }
+
}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowBackfillTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowBackfillTestCase.java
index 3e02673d12..e3315f4e72 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowBackfillTestCase.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowBackfillTestCase.java
@@ -17,48 +17,36 @@
package org.apache.dolphinscheduler.server.master.integration.cases;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition;
import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
import org.apache.dolphinscheduler.extract.master.command.BackfillWorkflowCommandParam;
import org.apache.dolphinscheduler.server.master.AbstractMasterIntegrationTestCase;
-import org.apache.dolphinscheduler.server.master.integration.Repository;
import org.apache.dolphinscheduler.server.master.integration.WorkflowOperator;
import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContext;
-import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContextFactory;
import org.apache.commons.lang3.time.DateUtils;
import java.time.Duration;
import java.util.List;
-import org.assertj.core.api.Assertions;
import org.assertj.core.util.Lists;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
/**
* The integration test for scheduling a workflow from workflow definition.
*/
public class WorkflowBackfillTestCase extends AbstractMasterIntegrationTestCase {
- @Autowired
- private WorkflowTestCaseContextFactory workflowTestCaseContextFactory;
-
- @Autowired
- private WorkflowOperator workflowOperator;
-
- @Autowired
- private Repository repository;
-
@Test
@DisplayName("Test backfill a workflow in asc order success")
public void testSerialBackfillWorkflow_with_oneSuccessTask() {
final String yaml = "/it/backfill/workflow_with_one_fake_task_success.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
final BackfillWorkflowCommandParam backfillWorkflowCommandParam = BackfillWorkflowCommandParam.builder()
.backfillTimeList(
@@ -78,22 +66,17 @@ public class WorkflowBackfillTestCase extends AbstractMasterIntegrationTestCase
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
final List<WorkflowInstance> workflowInstances = repository.queryWorkflowInstance(workflow);
- Assertions
- .assertThat(workflowInstances)
- .hasSize(4);
- Assertions
- .assertThat(workflowInstances.get(0).getScheduleTime())
+ assertThat(workflowInstances).hasSize(4);
+ assertThat(workflowInstances.get(0).getScheduleTime())
.isEqualTo(DateUtils.parseDate("2024-08-11 00:00:00", "yyyy-MM-dd HH:mm:ss"));
- Assertions
- .assertThat(workflowInstances.get(1).getScheduleTime())
+ assertThat(workflowInstances.get(1).getScheduleTime())
.isEqualTo(DateUtils.parseDate("2024-08-12 00:00:00", "yyyy-MM-dd HH:mm:ss"));
- Assertions
- .assertThat(workflowInstances.get(2).getScheduleTime())
+ assertThat(workflowInstances.get(2).getScheduleTime())
.isEqualTo(DateUtils.parseDate("2024-08-13 00:00:00", "yyyy-MM-dd HH:mm:ss"));
- Assertions
- .assertThat(workflowInstances.get(3).getScheduleTime())
+ assertThat(workflowInstances.get(3).getScheduleTime())
.isEqualTo(DateUtils.parseDate("2024-08-14 00:00:00", "yyyy-MM-dd HH:mm:ss"));
});
+ masterContainer.assertAllResourceReleased();
}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java
index 8130fec898..43571d2b73 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceFailoverTestCase.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.master.integration.cases;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.common.enums.Flag;
@@ -27,9 +28,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.server.master.AbstractMasterIntegrationTestCase;
import org.apache.dolphinscheduler.server.master.engine.system.SystemEventBus;
import org.apache.dolphinscheduler.server.master.engine.system.event.GlobalMasterFailoverEvent;
-import org.apache.dolphinscheduler.server.master.integration.Repository;
import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContext;
-import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContextFactory;
import org.apache.commons.lang3.StringUtils;
@@ -37,134 +36,113 @@ import java.time.Duration;
import java.util.Date;
import java.util.List;
-import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
public class WorkflowInstanceFailoverTestCase extends AbstractMasterIntegrationTestCase {
- @Autowired
- private WorkflowTestCaseContextFactory workflowTestCaseContextFactory;
-
@Autowired
private SystemEventBus systemEventBus;
- @Autowired
- private Repository repository;
-
@Test
public void testGlobalFailover_runningWorkflow_withSubmittedTasks() {
final String yaml = "/it/failover/running_workflowInstance_with_one_submitted_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.SUCCESS);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS, "state should success")
.matches(t -> t.getFlag() == Flag.YES)
.matches(t -> StringUtils.isNotEmpty(t.getLogPath()));
});
+ masterContainer.assertAllResourceReleased();
+
}
@Test
public void testGlobalFailover_runningWorkflow_withDispatchTasks() {
final String yaml = "/it/failover/running_workflowInstance_with_one_dispatched_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.SUCCESS);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(2);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE)
.matches(t -> t.getFlag() == Flag.NO);
- Assertions
- .assertThat(taskInstances.get(1))
+ assertThat(taskInstances.get(1))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS)
.matches(t -> t.getFlag() == Flag.YES)
.matches(t -> StringUtils.isNotEmpty(t.getLogPath()));
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_runningWorkflow_withRunningTasks() {
final String yaml = "/it/failover/running_workflowInstance_with_one_running_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.SUCCESS);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(2);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE)
.matches(t -> t.getFlag() == Flag.NO);
- Assertions
- .assertThat(taskInstances.get(1))
+ assertThat(taskInstances.get(1))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS)
.matches(t -> t.getFlag() == Flag.YES)
.matches(t -> StringUtils.isNotEmpty(t.getLogPath()));
});
+ masterContainer.assertAllResourceReleased();
}
@@ -172,111 +150,96 @@ public class WorkflowInstanceFailoverTestCase extends AbstractMasterIntegrationT
public void testGlobalFailover_runningWorkflow_withSuccessTasks() {
final String yaml = "/it/failover/running_workflowInstance_with_one_success_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.SUCCESS);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_runningWorkflow_withFailedTasks() {
final String yaml = "/it/failover/running_workflowInstance_with_one_failed_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.FAILURE);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.FAILURE)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyPauseWorkflow_withSubmittedTasks() {
final String yaml = "/it/failover/readyPause_workflowInstance_with_one_submitted_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.PAUSE);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.PAUSE)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyPauseWorkflow_withDispatchedTasks() {
final String yaml = "/it/failover/readyPause_workflowInstance_with_one_dispatched_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
@@ -286,180 +249,154 @@ public class WorkflowInstanceFailoverTestCase extends AbstractMasterIntegrationT
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.PAUSE);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(2);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE)
.matches(t -> t.getFlag() == Flag.NO);
- Assertions
- .assertThat(taskInstances.get(1))
+ assertThat(taskInstances.get(1))
.matches(t -> t.getState() == TaskExecutionStatus.PAUSE)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyPauseWorkflow_withSuccessTasks() {
final String yaml = "/it/failover/readyPause_workflowInstance_with_one_success_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.SUCCESS);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyPauseWorkflow_withFailedTasks() {
final String yaml = "/it/failover/readyPause_workflowInstance_with_one_failed_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.FAILURE);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.FAILURE)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyPauseWorkflow_withPausedTasks() {
final String yaml = "/it/failover/readyPause_workflowInstance_with_one_paused_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.PAUSE);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.PAUSE)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyStopWorkflow_withSubmittedTasks() {
final String yaml = "/it/failover/readyStop_workflowInstance_with_one_submitted_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.STOP);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.KILL)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyStopWorkflow_withDispatchedTasks() {
final String yaml = "/it/failover/readyStop_workflowInstance_with_one_dispatched_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
@@ -469,138 +406,117 @@ public class WorkflowInstanceFailoverTestCase extends AbstractMasterIntegrationT
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.STOP);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(2);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.NEED_FAULT_TOLERANCE)
.matches(t -> t.getFlag() == Flag.NO);
- Assertions
- .assertThat(taskInstances.get(1))
+ assertThat(taskInstances.get(1))
.matches(t -> t.getState() == TaskExecutionStatus.KILL)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyStopWorkflow_withSuccessTasks() {
final String yaml = "/it/failover/readyStop_workflowInstance_with_one_success_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.SUCCESS);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyStopWorkflow_withFailedTasks() {
final String yaml = "/it/failover/readyStop_workflowInstance_with_one_failed_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.FAILURE);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.FAILURE)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
@Test
public void testGlobalFailover_readyStopWorkflow_withKilledTasks() {
final String yaml = "/it/failover/readyStop_workflowInstance_with_one_killed_fake_task.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
systemEventBus.publish(GlobalMasterFailoverEvent.of(new Date()));
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflow))
+ assertThat(repository.queryWorkflowInstance(workflow))
.hasSize(1)
.anySatisfy(workflowInstance -> {
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.STOP);
- Assertions
- .assertThat(workflowInstance.getName())
+ assertThat(workflowInstance.getName())
.isEqualTo("workflow_with_one_fake_task_success-20240816071251690");
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflow);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(1);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> t.getState() == TaskExecutionStatus.KILL)
.matches(t -> t.getFlag() == Flag.YES);
});
-
+ masterContainer.assertAllResourceReleased();
}
}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java
index f45eb520be..fd9eee60f4 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstancePauseTestCase.java
@@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.server.master.integration.cases;
-import static com.google.common.truth.Truth.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.common.enums.Flag;
@@ -27,43 +27,26 @@ import org.apache.dolphinscheduler.dao.entity.WorkflowDefinition;
import org.apache.dolphinscheduler.extract.master.command.RunWorkflowCommandParam;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.server.master.AbstractMasterIntegrationTestCase;
-import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
-import org.apache.dolphinscheduler.server.master.integration.Repository;
import org.apache.dolphinscheduler.server.master.integration.WorkflowOperator;
import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContext;
-import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContextFactory;
import java.time.Duration;
import java.util.List;
-import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
/**
* The integration test for pausing a workflow instance.
*/
public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTestCase {
- @Autowired
- private WorkflowTestCaseContextFactory workflowTestCaseContextFactory;
-
- @Autowired
- private WorkflowOperator workflowOperator;
-
- @Autowired
- private IWorkflowRepository workflowRepository;
-
- @Autowired
- private Repository repository;
-
@Test
@DisplayName("Test pause a workflow with one success task")
public void testPauseWorkflow_with_oneSuccessTask() {
final String yaml = "/it/pause/workflow_with_one_fake_task_success.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder()
.workflowDefinition(workflow)
@@ -75,38 +58,33 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
.pollInterval(Duration.ofMillis(100))
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION);
});
- Assertions
- .assertThat(repository.queryTaskInstance(workflowInstanceId))
+ assertThat(repository.queryTaskInstance(workflowInstanceId))
.satisfiesExactly(taskInstance -> {
assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.RUNNING_EXECUTION);
});
});
- assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess());
+ assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess()).isTrue();
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState()).isEqualTo(WorkflowExecutionStatus.SUCCESS);
});
- Assertions
- .assertThat(repository.queryTaskInstance(workflowInstanceId))
+ assertThat(repository.queryTaskInstance(workflowInstanceId))
.satisfiesExactly(taskInstance -> {
assertThat(taskInstance.getName()).isEqualTo("A");
assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.SUCCESS);
});
});
-
- assertThat(workflowRepository.getAll()).isEmpty();
+ masterContainer.assertAllResourceReleased();
}
@Test
@@ -114,7 +92,7 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
public void testPauseWorkflow_with_oneFailedTask() {
final String yaml = "/it/pause/workflow_with_one_fake_task_failed.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder()
.workflowDefinition(workflow)
@@ -126,38 +104,35 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
.pollInterval(Duration.ofMillis(100))
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION);
});
- Assertions
- .assertThat(
- repository.queryTaskInstance(workflowInstanceId))
- .satisfiesExactly(taskInstance -> {
- assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.RUNNING_EXECUTION);
- });
+ assertThat(
+ repository.queryTaskInstance(workflowInstanceId))
+ .satisfiesExactly(taskInstance -> {
+ assertThat(taskInstance.getState())
+ .isEqualTo(TaskExecutionStatus.RUNNING_EXECUTION);
+ });
});
- assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess());
+ assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess()).isTrue();
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState()).isEqualTo(WorkflowExecutionStatus.FAILURE);
});
- Assertions
- .assertThat(repository.queryTaskInstance(workflowInstanceId))
+ assertThat(repository.queryTaskInstance(workflowInstanceId))
.satisfiesExactly(taskInstance -> {
assertThat(taskInstance.getName()).isEqualTo("A");
assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.FAILURE);
});
});
- assertThat(workflowRepository.getAll()).isEmpty();
+ masterContainer.assertAllResourceReleased();
}
@Test
@@ -165,7 +140,7 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
public void testPauseWorkflow_with_threeParallelSuccessTask() {
final String yaml = "/it/pause/workflow_with_three_parallel_three_fake_task_success.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder()
.workflowDefinition(workflow)
@@ -174,16 +149,21 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
final Integer workflowInstanceId = workflowOperator.manualTriggerWorkflow(workflowTriggerDTO);
await()
- .pollInterval(Duration.ofMillis(100))
+ .pollInterval(Duration.ofMillis(50))
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions.assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION);
});
- Assertions
- .assertThat(repository.queryTaskInstance(workflowInstanceId))
+ });
+
+ await()
+ .pollInterval(Duration.ofMillis(50))
+ .atMost(Duration.ofMinutes(1))
+ .untilAsserted(() -> {
+ assertThat(repository.queryTaskInstance(workflowInstanceId))
.anySatisfy(taskInstance -> {
assertThat(taskInstance.getName()).isEqualTo("A1");
assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.RUNNING_EXECUTION);
@@ -198,18 +178,16 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
});
});
- assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess());
+ assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess()).isTrue();
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState()).isEqualTo(WorkflowExecutionStatus.PAUSE);
});
- Assertions
- .assertThat(repository.queryTaskInstance(workflowInstanceId))
+ assertThat(repository.queryTaskInstance(workflowInstanceId))
.hasSize(6)
.anySatisfy(taskInstance -> {
assertThat(taskInstance.getName()).isEqualTo("A1");
@@ -236,7 +214,7 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.PAUSE);
});
});
- assertThat(workflowRepository.getAll()).isEmpty();
+ masterContainer.assertAllResourceReleased();
}
@Test
@@ -244,7 +222,7 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
public void testPauseWorkflow_with_subWorkflowTask_success() {
final String yaml = "/it/pause/workflow_with_sub_workflow_task_success.yaml";
final WorkflowTestCaseContext context = workflowTestCaseContextFactory.initializeContextFromYaml(yaml);
- final WorkflowDefinition workflow = context.getWorkflows().get(0);
+ final WorkflowDefinition workflow = context.getOneWorkflow();
final WorkflowOperator.WorkflowTriggerDTO workflowTriggerDTO = WorkflowOperator.WorkflowTriggerDTO.builder()
.workflowDefinition(workflow)
@@ -252,69 +230,68 @@ public class WorkflowInstancePauseTestCase extends AbstractMasterIntegrationTest
.build();
final Integer workflowInstanceId = workflowOperator.manualTriggerWorkflow(workflowTriggerDTO);
+ final WorkflowDefinition subWorkflowDefinition = context.getWorkflows().get(1);
await()
- .pollInterval(Duration.ofMillis(100))
+ .pollInterval(Duration.ofMillis(50))
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION);
});
- Assertions
- .assertThat(repository.queryTaskInstance(workflowInstanceId))
+ assertThat(repository.queryTaskInstance(workflowInstanceId))
.satisfiesExactly(taskInstance -> {
- assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.RUNNING_EXECUTION);
+ assertThat(taskInstance.getState())
+ .isEqualTo(TaskExecutionStatus.RUNNING_EXECUTION);
});
- Assertions
- .assertThat(repository.queryWorkflowInstance(context.getWorkflows().get(1)))
+ // Wait the sub workflow instance's task is running, then pause it
+ // Otherwise the subworkflow instance's first task might be paused
+ assertThat(repository.queryWorkflowInstance(subWorkflowDefinition))
.satisfiesExactly(workflowInstance -> {
assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION);
});
+ assertThat(repository.queryTaskInstance(subWorkflowDefinition))
+ .satisfiesExactly(taskInstance -> {
+ assertThat(taskInstance.getState())
+ .isEqualTo(TaskExecutionStatus.RUNNING_EXECUTION);
+ });
});
- assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess());
+ assertThat(workflowOperator.pauseWorkflowInstance(workflowInstanceId).isSuccess()).isTrue();
await()
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId))
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId))
.satisfies(workflowInstance -> {
assertThat(workflowInstance.getState()).isEqualTo(WorkflowExecutionStatus.PAUSE);
assertThat(workflowInstance.getIsSubWorkflow()).isEqualTo(Flag.NO);
});
- Assertions
- .assertThat(repository.queryTaskInstance(workflowInstanceId))
+ assertThat(repository.queryTaskInstance(workflowInstanceId))
.satisfiesExactly(taskInstance -> {
assertThat(taskInstance.getName()).isEqualTo("sub_logic_task");
assertThat(taskInstance.getState()).isEqualTo(TaskExecutionStatus.PAUSE);
});
- final WorkflowDefinition subWorkflowDefinition = context.getWorkflows().get(1);
- Assertions
- .assertThat(repository.queryWorkflowInstance(subWorkflowDefinition))
+ assertThat(repository.queryWorkflowInstance(subWorkflowDefinition))
.satisfiesExactly(workflowInstance -> {
assertThat(workflowInstance.getState()).isEqualTo(WorkflowExecutionStatus.PAUSE);
assertThat(workflowInstance.getIsSubWorkflow()).isEqualTo(Flag.YES);
});
final List<TaskInstance> taskInstances = repository.queryTaskInstance(subWorkflowDefinition);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(2);
- Assertions
- .assertThat(taskInstances.get(0).getState())
+ assertThat(taskInstances.get(0).getState())
.isEqualTo(TaskExecutionStatus.SUCCESS);
- Assertions
- .assertThat(taskInstances.get(1).getState())
+ assertThat(taskInstances.get(1).getState())
.isEqualTo(TaskExecutionStatus.PAUSE);
});
- assertThat(workflowRepository.getAll()).isEmpty();
+ masterContainer.assertAllResourceReleased();
}
}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverFailureTaskTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverFailureTaskTestCase.java
index 66e64081a3..c10b481e0c 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverFailureTaskTestCase.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverFailureTaskTestCase.java
@@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.server.master.integration.cases;
-import static com.google.common.truth.Truth.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.common.enums.Flag;
@@ -26,39 +26,21 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.server.master.AbstractMasterIntegrationTestCase;
-import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
-import org.apache.dolphinscheduler.server.master.integration.Repository;
-import org.apache.dolphinscheduler.server.master.integration.WorkflowOperator;
import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContext;
-import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContextFactory;
import org.apache.commons.lang3.StringUtils;
import java.time.Duration;
import java.util.List;
-import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
/**
* The integration test for recover from failure tasks.
*/
public class WorkflowInstanceRecoverFailureTaskTestCase extends AbstractMasterIntegrationTestCase {
- @Autowired
- private WorkflowTestCaseContextFactory workflowTestCaseContextFactory;
-
- @Autowired
- private WorkflowOperator workflowOperator;
-
- @Autowired
- private IWorkflowRepository workflowRepository;
-
- @Autowired
- private Repository repository;
-
@Test
@DisplayName("Test recover from failure tasks")
public void testRepeatRunningWorkflow_with_taskOnly() {
@@ -72,8 +54,7 @@ public class WorkflowInstanceRecoverFailureTaskTestCase extends AbstractMasterIn
.pollInterval(Duration.ofMillis(100))
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
- Assertions
- .assertThat(repository.queryWorkflowInstance(workflowInstanceId).getState())
+ assertThat(repository.queryWorkflowInstance(workflowInstanceId).getState())
.isEqualTo(WorkflowExecutionStatus.RUNNING_EXECUTION);
});
@@ -81,37 +62,31 @@ public class WorkflowInstanceRecoverFailureTaskTestCase extends AbstractMasterIn
.atMost(Duration.ofMinutes(1))
.untilAsserted(() -> {
final WorkflowInstance workflowInstance = repository.queryWorkflowInstance(workflowInstanceId);
- Assertions
- .assertThat(workflowInstance.getState())
+ assertThat(workflowInstance.getState())
.isEqualTo(WorkflowExecutionStatus.SUCCESS);
- Assertions
- .assertThat(workflowInstance.getRunTimes())
+ assertThat(workflowInstance.getRunTimes())
.isEqualTo(2);
final List<TaskInstance> taskInstances = repository.queryTaskInstance(workflowInstanceId);
- Assertions
- .assertThat(taskInstances)
+ assertThat(taskInstances)
.hasSize(3);
- Assertions
- .assertThat(taskInstances.get(0))
+ assertThat(taskInstances.get(0))
.matches(t -> "A".equals(t.getName()))
.matches(t -> t.getState() == TaskExecutionStatus.FAILURE)
.matches(t -> t.getFlag() == Flag.NO);
- Assertions
- .assertThat(taskInstances.get(1))
+ assertThat(taskInstances.get(1))
.matches(t -> "A".equals(t.getName()))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS)
.matches(t -> t.getFlag() == Flag.YES)
.matches(t -> StringUtils.isNotEmpty(t.getLogPath()));
- Assertions
- .assertThat(taskInstances.get(2))
+ assertThat(taskInstances.get(2))
.matches(t -> t.getState() == TaskExecutionStatus.SUCCESS)
.matches(t -> t.getFlag() == Flag.YES)
.matches(t -> StringUtils.isNotEmpty(t.getLogPath()));
});
- assertThat(workflowRepository.getAll()).isEmpty();
+ masterContainer.assertAllResourceReleased();
}
}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverPauseTestCase.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverPauseTestCase.java
index 14a53aae7f..c21dc8d6b4 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverPauseTestCase.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/integration/cases/WorkflowInstanceRecoverPauseTestCase.java
@@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.server.master.integration.cases;
-import static com.google.common.truth.Truth.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
import org.apache.dolphinscheduler.common.enums.Flag;
@@ -27,43 +27,26 @@ import org.apache.dolphinscheduler.dao.entity.WorkflowInstance;
import org.apache.dolphinscheduler.extract.master.command.RunWorkflowCommandParam;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.server.master.AbstractMasterIntegrationTestCase;
-import org.apache.dolphinscheduler.server.master.engine.IWorkflowRepository;
-import org.apache.dolphinscheduler.server.master.integration.Repository;
import org.apache.dolphinscheduler.server.master.integration.WorkflowOperator;
import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContext;
-import org.apache.dolphinscheduler.server.master.integration.WorkflowTestCaseContextFactory;
import java.time.Duration;
import java.util.List;
-import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
/**
* The integration test for pausing a workflow instance.
*/
public class WorkflowInstanceRecoverPauseTestCase extends AbstractMasterIntegrationTestCase {
- @Autowired
- private WorkflowTestCaseContextFactory workflowTestCaseContextFactory;
-
- @Autowired
- private WorkflowOperator workflowOperator;
... 9906 lines suppressed ...