You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2024/02/06 10:25:47 UTC
(dolphinscheduler) branch 3.2.1-prepare updated (898b001608 -> 58aba58079)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a change to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
from 898b001608 cp: Reduce the size of tarball to continue ASF release (#15004) (#15540)
new be58ba69a9 Exit JVM when OOM (#15538)
new d9529d95f3 refactor comments & function name for confuse (#15546)
new c8f120a60a Fix TaskGroupQueue will never be wakeup due to wakeup failed at one time (#15528)
new 9e4e890f84 [Doc][Docker] fix typo on start with docker (#15534)
new a76b4662be Fix createFile with permission will not work (#15556)
new acb1e0a09f fix: data quality can not use (#15551)
new ca93aa8e26 fix switch js (#15487)
new 19adf25920 [Bug][Task Api] fix 'MACPATTERN' in ProcessUtils and cover all cases on MacOS in ProcessUtilsTest (#15480) (#15529)
new d87a1acd19 fix: ddl without drop exists (#14128)
new 7d660ec2f9 [Bug][force-success] force success add end time (#15144)
new fa7b4c1c8f fix: start param for wf not work (#15544)
new e2d08fc27d fix: data quality may fail in docker mode (#15563)
new 52d315d3ca Fix Recover WorkflowInstance will casue workflow Instance state is success but task insatnce is killed/paused (#15574)
new 8e9537ad33 [Improvement][UT] Improve Worker registry coverage (#15380)
new eff7364ee1 Fix create parent directory will cause FileAlreadyExistsException (#15576)
new 0b722999d3 [Fix] Fix WorkflowInstance batch start failed will throw incorrect exception. (#15577)
new 9bb526855d Set the workflow instance ready state to running in failover (#15572)
new 58aba58079 [fix][worker]:Fix the issue of missing disk usage reporting information in the worker (#15565)
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
deploy/kubernetes/dolphinscheduler/README.md | 2 +-
deploy/kubernetes/dolphinscheduler/values.yaml | 2 +-
docs/docs/en/architecture/configuration.md | 2 +-
docs/docs/en/guide/data-quality.md | 10 +-
docs/docs/en/guide/resource/configuration.md | 6 +-
docs/docs/en/guide/start/docker.md | 8 +-
docs/docs/en/guide/upgrade/incompatible.md | 2 +
docs/docs/zh/architecture/configuration.md | 2 +-
docs/docs/zh/guide/data-quality.md | 9 +-
docs/docs/zh/guide/resource/configuration.md | 6 +-
docs/docs/zh/guide/start/docker.md | 8 +-
.../src/main/bin/jvm_args_env.sh | 1 +
.../resources/docker/file-manage/common.properties | 6 +-
dolphinscheduler-api/src/main/bin/jvm_args_env.sh | 1 +
.../api/controller/ExecutorController.java | 8 +-
.../api/controller/K8sNamespaceController.java | 4 +-
.../api/controller/TaskGroupController.java | 26 +-
.../api/service/K8sNamespaceService.java | 4 +-
.../api/service/TaskGroupService.java | 8 -
.../api/service/impl/ExecutorServiceImpl.java | 49 +--
.../api/service/impl/K8SNamespaceServiceImpl.java | 4 +-
.../service/impl/TaskGroupQueueServiceImpl.java | 18 +-
.../api/service/impl/TaskGroupServiceImpl.java | 23 +-
.../api/service/impl/TaskInstanceServiceImpl.java | 1 +
.../api/service/K8SNamespaceServiceTest.java | 6 +-
.../api/service/ProcessInstanceServiceTest.java | 1 -
.../api/service/TaskGroupServiceTest.java | 9 +-
.../common/constants/DataSourceConstants.java | 2 +-
.../dolphinscheduler/common/utils/FileUtils.java | 145 ++-----
.../src/main/resources/common.properties | 6 +-
.../common/log/SensitiveDataConverterTest.java | 28 ++
.../common/utils/FileUtilsTest.java | 24 +-
.../src/test/resources/common.properties | 6 +-
.../dolphinscheduler/dao/entity/TaskGroup.java | 33 +-
.../dao/mapper/TaskGroupMapper.java | 32 +-
.../dao/mapper/TaskGroupQueueMapper.java | 17 +
.../dao/mapper/TaskInstanceMapper.java | 33 --
.../dao/repository/TaskGroupDao.java | 62 +++
.../dao/repository/TaskGroupQueueDao.java | 64 +++
.../dao/repository/TaskInstanceDao.java | 5 +-
.../dao/repository/impl/TaskGroupDaoImpl.java | 70 ++++
.../dao/repository/impl/TaskGroupQueueDaoImpl.java | 73 ++++
.../dao/mapper/TaskGroupMapper.xml | 58 ++-
.../dao/mapper/TaskGroupQueueMapper.xml | 61 ++-
.../dao/mapper/TaskInstanceMapper.xml | 99 +----
.../src/main/resources/sql/dolphinscheduler_h2.sql | 1 +
.../main/resources/sql/dolphinscheduler_mysql.sql | 5 +
.../resources/sql/dolphinscheduler_postgresql.sql | 2 +
.../3.2.1_schema/mysql/dolphinscheduler_ddl.sql | 17 +-
.../postgresql/dolphinscheduler_ddl.sql | 4 +-
.../3.3.0_schema/mysql/dolphinscheduler_ddl.sql | 17 +-
.../postgresql/dolphinscheduler_ddl.sql | 4 +-
.../dao/mapper/ProcessInstanceMapperTest.java | 24 --
.../dao/mapper/TaskGroupMapperTest.java | 9 +-
.../dao/mapper/TaskInstanceMapperTest.java | 110 -----
.../dao/repository/impl/TaskGroupDaoImplTest.java | 118 ++++++
.../repository/impl/TaskGroupQueueDaoImplTest.java | 108 +++++
.../plugin/datasource/api/utils/CommonUtils.java | 78 +++-
.../mysql/param/MySQLDataSourceProcessor.java | 14 +
.../resources/docker/file-manage/common.properties | 6 +-
.../extract/master/ILogicTaskInstanceOperator.java | 9 -
.../extract/master/IWorkflowInstanceService.java | 5 +
.../transportor/TaskInstanceForceStartRequest.java | 42 --
.../TaskInstanceForceStartResponse.java | 41 --
.../transportor/TaskInstanceWakeupRequest.java | 19 +-
.../src/main/bin/jvm_args_env.sh | 1 +
.../server/master/MasterServer.java | 5 +
.../master/event/StateEventHandlerManager.java | 10 +-
.../server/master/event/TaskStateEventHandler.java | 11 +-
.../event/TaskWaitTaskGroupStateHandler.java | 47 ---
.../LogicTaskInstanceOperationFunctionManager.java | 13 -
.../master/rpc/LogicTaskInstanceOperatorImpl.java | 14 -
.../TaskInstanceForceStartOperationFunction.java | 58 ---
.../rpc/TaskInstanceWakeupOperationFunction.java | 36 +-
.../master/rpc/WorkflowInstanceServiceImpl.java | 10 +
.../master/runner/TaskExecutionContextFactory.java | 27 +-
.../master/runner/WorkflowExecuteRunnable.java | 186 ++-------
.../runner/WorkflowExecuteRunnableFactory.java | 9 +-
.../runner/taskgroup/TaskGroupCoordinator.java | 451 +++++++++++++++++++++
.../server/master/utils/SwitchTaskUtils.java | 17 +
.../master/runner/WorkflowExecuteRunnableTest.java | 11 +-
.../runner/taskgroup/TaskGroupCoordinatorTest.java | 182 +++++++++
.../server/master/utils/SwitchTaskUtilsTest.java | 14 +
.../registry/api/enums/RegistryNodeType.java | 1 +
.../service/expand/CuringParamsService.java | 12 +
.../service/expand/CuringParamsServiceImpl.java | 27 +-
.../service/process/ProcessService.java | 37 +-
.../service/process/ProcessServiceImpl.java | 415 ++-----------------
.../service/utils/ProcessUtils.java | 3 +-
.../service/expand/CuringParamsServiceTest.java | 44 ++
.../service/process/ProcessServiceTest.java | 16 -
.../spi/datasource/DefaultConnectionParam.java | 7 +-
.../src/main/bin/jvm_args_env.sh | 1 +
.../src/main/docker/Dockerfile | 1 +
.../plugin/storage/abs/AbsStorageOperator.java | 3 +-
.../plugin/storage/gcs/GcsStorageOperator.java | 3 +-
.../plugin/storage/obs/ObsStorageOperator.java | 3 +-
.../plugin/storage/oss/OssStorageOperator.java | 3 +-
.../plugin/storage/s3/S3StorageOperator.java | 3 +-
.../plugin/task/api/TaskConstants.java | 4 +-
.../plugin/task/api/model/JdbcInfo.java | 73 +---
.../shell/BaseLinuxShellInterceptorBuilder.java | 8 +-
.../plugin/task/api/utils/JdbcUrlParser.java | 24 +-
.../plugin/task/api/utils/ProcessUtils.java | 2 +-
.../plugin/task/api/utils/JdbcUrlParserTest.java | 18 +-
.../plugin/task/api/utils/ProcessUtilsTest.java | 3 +
.../src/test/resources/common.properties | 6 +-
.../plugin/task/dq/DataQualityTask.java | 6 +-
.../plugin/task/java/JavaTaskTest.java | 3 +-
.../src/main/bin/jvm_args_env.sh | 1 +
.../server/worker/registry/WorkerStopStrategy.java | 3 -
.../worker/registry/WorkerWaitingStrategy.java | 11 +
.../server/worker/task/WorkerHeartBeatTask.java | 1 +
.../worker/utils/TaskExecutionContextUtils.java | 3 +-
.../WorkerConnectionStateListenerTest.java | 60 +++
.../worker/registry/WorkerRegistryClientTest.java | 40 +-
.../server/worker/registry/WorkerStrategyTest.java | 187 +++++++++
.../utils/TaskExecutionContextUtilsTest.java | 2 +-
118 files changed, 2233 insertions(+), 1580 deletions(-)
create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupDao.java
create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupQueueDao.java
create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImpl.java
create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImpl.java
create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImplTest.java
create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java
delete mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java
delete mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java
delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
delete mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceForceStartOperationFunction.java
create mode 100644 dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java
create mode 100644 dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinatorTest.java
copy dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-doris/src/main/java/org/apache/dolphinscheduler/plugin/doris/param/DorisConnectionParam.java => dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DefaultConnectionParam.java (85%)
create mode 100644 dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListenerTest.java
create mode 100644 dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStrategyTest.java
(dolphinscheduler) 02/18: refactor comments & function name for confuse (#15546)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit d9529d95f3ee8b1a604483a880a698eb51b9a836
Author: John Huang <pe...@gmail.com>
AuthorDate: Thu Feb 1 10:39:54 2024 +0800
refactor comments & function name for confuse (#15546)
Co-authored-by: fuchanghai <ch...@apache.org>
(cherry picked from commit 2f66a667ba6bed20003cfb17af3f347166e678c1)
---
.../dolphinscheduler/api/controller/K8sNamespaceController.java | 4 ++--
.../apache/dolphinscheduler/api/service/K8sNamespaceService.java | 4 ++--
.../dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java | 4 ++--
.../dolphinscheduler/api/service/K8SNamespaceServiceTest.java | 6 +++---
4 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java
index f150f0c015..38b1923c82 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/K8sNamespaceController.java
@@ -92,7 +92,7 @@ public class K8sNamespaceController extends BaseController {
}
/**
- * create namespace,if not exist on k8s,will create,if exist only register in db
+ * register namespace in db,need to create namespace in k8s first
*
* @param loginUser
* @param namespace k8s namespace
@@ -111,7 +111,7 @@ public class K8sNamespaceController extends BaseController {
@RequestParam(value = "namespace") String namespace,
@RequestParam(value = "clusterCode") Long clusterCode) {
Map<String, Object> result =
- k8sNamespaceService.createK8sNamespace(loginUser, namespace, clusterCode);
+ k8sNamespaceService.registerK8sNamespace(loginUser, namespace, clusterCode);
return returnDataList(result);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java
index daf988fdfe..032612a4ee 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/K8sNamespaceService.java
@@ -41,14 +41,14 @@ public interface K8sNamespaceService {
Result queryListPaging(User loginUser, String searchVal, Integer pageNo, Integer pageSize);
/**
- * create namespace,if not exist on k8s,will create,if exist only register in db
+ * register namespace in db,need to create namespace in k8s first
*
* @param loginUser login user
* @param namespace namespace
* @param clusterCode k8s not null
* @return
*/
- Map<String, Object> createK8sNamespace(User loginUser, String namespace, Long clusterCode);
+ Map<String, Object> registerK8sNamespace(User loginUser, String namespace, Long clusterCode);
/**
* verify namespace and k8s
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java
index f05e69b9db..06f3608207 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/K8SNamespaceServiceImpl.java
@@ -100,7 +100,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames
}
/**
- * create namespace,if not exist on k8s,will create,if exist only register in db
+ * register namespace in db,need to create namespace in k8s first
*
* @param loginUser login user
* @param namespace namespace
@@ -108,7 +108,7 @@ public class K8SNamespaceServiceImpl extends BaseServiceImpl implements K8sNames
* @return
*/
@Override
- public Map<String, Object> createK8sNamespace(User loginUser, String namespace, Long clusterCode) {
+ public Map<String, Object> registerK8sNamespace(User loginUser, String namespace, Long clusterCode) {
Map<String, Object> result = new HashMap<>();
if (isNotAdmin(loginUser)) {
throw new ServiceException(Status.USER_NO_OPERATION_PERM);
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/K8SNamespaceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/K8SNamespaceServiceTest.java
index b4b95ea497..6cc6f0d324 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/K8SNamespaceServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/K8SNamespaceServiceTest.java
@@ -107,16 +107,16 @@ public class K8SNamespaceServiceTest {
public void createK8sNamespace() {
// namespace is null
Map<String, Object> result =
- k8sNamespaceService.createK8sNamespace(getLoginUser(), null, clusterCode);
+ k8sNamespaceService.registerK8sNamespace(getLoginUser(), null, clusterCode);
logger.info(result.toString());
Assertions.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
// k8s is null
- result = k8sNamespaceService.createK8sNamespace(getLoginUser(), namespace, null);
+ result = k8sNamespaceService.registerK8sNamespace(getLoginUser(), namespace, null);
logger.info(result.toString());
Assertions.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS));
// correct
Mockito.when(clusterMapper.queryByClusterCode(Mockito.anyLong())).thenReturn(getCluster());
- result = k8sNamespaceService.createK8sNamespace(getLoginUser(), namespace, clusterCode);
+ result = k8sNamespaceService.registerK8sNamespace(getLoginUser(), namespace, clusterCode);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
(dolphinscheduler) 13/18: Fix Recover WorkflowInstance will casue workflow Instance state is success but task insatnce is killed/paused (#15574)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 52d315d3cabbcb3aa27c83b3730b5db77adda4a1
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Feb 6 10:09:30 2024 +0800
Fix Recover WorkflowInstance will casue workflow Instance state is success but task insatnce is killed/paused (#15574)
(cherry picked from commit 43a06525a29e01f6769aa0057512884d7441b438)
---
.../api/service/ProcessInstanceServiceTest.java | 1 -
.../dao/mapper/TaskInstanceMapper.java | 33 -------
.../dao/repository/TaskInstanceDao.java | 5 +-
.../dao/mapper/TaskInstanceMapper.xml | 99 +------------------
.../dao/mapper/ProcessInstanceMapperTest.java | 24 -----
.../dao/mapper/TaskInstanceMapperTest.java | 110 ---------------------
.../service/process/ProcessService.java | 3 -
.../service/process/ProcessServiceImpl.java | 61 ++++--------
8 files changed, 26 insertions(+), 310 deletions(-)
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
index 800f8bda18..9fb4d83052 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessInstanceServiceTest.java
@@ -769,7 +769,6 @@ public class ProcessInstanceServiceTest {
processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion())).thenReturn(new ProcessDefinitionLog());
when(processInstanceMapper.queryDetailById(1)).thenReturn(processInstance);
- when(taskInstanceMapper.queryByInstanceIdAndName(Mockito.anyInt(), Mockito.any())).thenReturn(taskInstance);
DAG<Long, TaskNode, TaskNodeRelation> graph = new DAG<>();
for (long i = 1; i <= 7; ++i) {
graph.addNode(i, new TaskNode());
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java
index 04e818f5c3..f80dac826c 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java
@@ -39,23 +39,10 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
*/
public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
- List<Integer> queryTaskByProcessIdAndState(@Param("processInstanceId") Integer processInstanceId,
- @Param("state") Integer state);
-
List<TaskInstance> findValidTaskListByProcessId(@Param("processInstanceId") Integer processInstanceId,
@Param("flag") Flag flag,
@Param("testFlag") int testFlag);
- List<TaskInstance> queryByHostAndStatus(@Param("host") String host,
- @Param("states") int[] stateArray);
-
- int setFailoverByHostAndStateArray(@Param("host") String host,
- @Param("states") int[] stateArray,
- @Param("destStatus") TaskExecutionStatus destStatus);
-
- TaskInstance queryByInstanceIdAndName(@Param("processInstanceId") int processInstanceId,
- @Param("name") String name);
-
TaskInstance queryByInstanceIdAndCode(@Param("processInstanceId") int processInstanceId,
@Param("taskCode") Long taskCode);
@@ -66,9 +53,6 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
List<TaskInstance> queryByProcessInstanceIdsAndTaskCodes(@Param("processInstanceIds") List<Integer> processInstanceIds,
@Param("taskCodes") List<Long> taskCodes);
- Integer countTask(@Param("projectCodes") Long[] projectCodes,
- @Param("taskIds") int[] taskIds);
-
/**
* Statistics task instance group by given project codes list by start time
* <p>
@@ -97,20 +81,6 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
@Param("endTime") Date endTime,
@Param("projectIds") Set<Integer> projectIds);
- /**
- * Statistics task instance group by given project codes list by submit time
- * <p>
- * We only need project codes to determine whether the task instance belongs to the user or not.
- *
- * @param startTime Statistics start time
- * @param endTime Statistics end time
- * @param projectCodes Project codes list to filter
- * @return List of ExecuteStatusCount
- */
- List<ExecuteStatusCount> countTaskInstanceStateByProjectCodesAndStatesBySubmitTime(@Param("startTime") Date startTime,
- @Param("endTime") Date endTime,
- @Param("projectCodes") Long[] projectCodes,
- @Param("states") List<TaskExecutionStatus> states);
/**
* Statistics task instance group by given project codes list by submit time
* <p>
@@ -159,9 +129,6 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
@Param("startTime") Date startTime,
@Param("endTime") Date endTime);
- List<TaskInstance> loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId,
- @Param("status") int status);
-
void deleteByWorkflowInstanceId(@Param("workflowInstanceId") int workflowInstanceId);
List<TaskInstance> findByWorkflowInstanceId(@Param("workflowInstanceId") Integer workflowInstanceId);
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskInstanceDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskInstanceDao.java
index 0156416fd3..27a0dbb0f6 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskInstanceDao.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskInstanceDao.java
@@ -100,9 +100,10 @@ public interface TaskInstanceDao extends IDao<TaskInstance> {
/**
* find last task instance corresponding to taskCode in the date interval
+ *
* @param processInstanceId Task's parent process instance id
- * @param depTaskCode taskCode
- * @param testFlag test flag
+ * @param depTaskCode taskCode
+ * @param testFlag test flag
* @return task instance
*/
TaskInstance queryLastTaskInstanceIntervalInProcessInstance(Integer processInstanceId,
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
index 1544d0ed8f..548baae566 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
@@ -30,24 +30,6 @@
${alias}.flag, ${alias}.is_cache, ${alias}.cache_key, ${alias}.retry_interval, ${alias}.max_retry_times, ${alias}.task_instance_priority, ${alias}.worker_group,${alias}.environment_code , ${alias}.executor_id,
${alias}.first_submit_time, ${alias}.delay_time, ${alias}.task_params, ${alias}.var_pool, ${alias}.dry_run, ${alias}.test_flag, ${alias}.task_group_id, ${alias}.task_execute_type
</sql>
- <update id="setFailoverByHostAndStateArray">
- update t_ds_task_instance
- set state = #{destStatus}
- where host = #{host}
- <if test="states != null and states.length != 0">
- and state in
- <foreach collection="states" index="index" item="i" open="(" separator="," close=")">
- #{i}
- </foreach>
- </if>
- </update>
- <select id="queryTaskByProcessIdAndState" resultType="java.lang.Integer">
- select id
- from t_ds_task_instance
- WHERE process_instance_id = #{processInstanceId}
- and state = #{state}
- and flag = 1
- </select>
<select id="findValidTaskListByProcessId" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
select
<include refid="baseSql"/>
@@ -63,21 +45,6 @@
from t_ds_task_instance
WHERE process_instance_id = #{workflowInstanceId}
</select>
- <select id="queryByHostAndStatus" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
- select
- <include refid="baseSql"/>
- from t_ds_task_instance
- where 1 = 1
- <if test="host != null and host != ''">
- and host = #{host}
- </if>
- <if test="states != null and states.length != 0">
- and state in
- <foreach collection="states" index="index" item="i" open="(" separator="," close=")">
- #{i}
- </foreach>
- </if>
- </select>
<select id="countTaskInstanceStateByProjectCodes" resultType="org.apache.dolphinscheduler.dao.model.TaskInstanceStatusCountDto">
select state, count(0) as count
@@ -118,32 +85,7 @@
</if>
group by t.state
</select>
- <select id="countTaskInstanceStateByProjectCodesAndStatesBySubmitTime" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
- select
- state, count(0) as count
- from t_ds_task_instance t
- left join t_ds_task_definition_log d on d.code=t.task_code and d.version=t.task_definition_version
- where 1=1
- <if test="states != null and states.size != 0">
- and t.state in
- <foreach collection="states" index="index" item="state" open="(" separator="," close=")">
- #{state.code}
- </foreach>
- </if>
- <if test="projectCodes != null and projectCodes.length != 0">
- and d.project_code in
- <foreach collection="projectCodes" index="index" item="i" open="(" separator="," close=")">
- #{i}
- </foreach>
- </if>
- <if test="startTime != null">
- and t.submit_time <![CDATA[ > ]]> #{startTime}
- </if>
- <if test="endTime != null">
- and t.submit_time <![CDATA[ <= ]]> #{endTime}
- </if>
- group by t.state
- </select>
+
<select id="countTaskInstanceStateByProjectCodesAndStatesBySubmitTimeV2" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
select
state, count(0) as count
@@ -181,15 +123,7 @@
</if>
group by t.state
</select>
- <select id="queryByInstanceIdAndName" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
- select
- <include refid="baseSql"/>
- from t_ds_task_instance
- where process_instance_id = #{processInstanceId}
- and name = #{name}
- and flag = 1
- limit 1
- </select>
+
<select id="queryByInstanceIdAndCode" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
select
<include refid="baseSql"/>
@@ -229,24 +163,7 @@
</foreach>
</if>
</select>
- <select id="countTask" resultType="java.lang.Integer">
- select count(1) as count
- from t_ds_task_instance task,t_ds_task_definition_log define
- where task.task_code=define.code
- and task.task_definition_version=define.version
- <if test="projectCodes != null and projectCodes.length != 0">
- and define.project_code in
- <foreach collection="projectCodes" index="index" item="i" open="(" separator="," close=")">
- #{i}
- </foreach>
- </if>
- <if test="taskIds != null and taskIds.length != 0">
- and task.id in
- <foreach collection="taskIds" index="index" item="i" open="(" separator="," close=")">
- #{i}
- </foreach>
- </if>
- </select>
+
<select id="queryTaskInstanceListPaging" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
select
<include refid="baseSql"/>
@@ -330,16 +247,6 @@
</if>
order by start_time desc
</select>
- <select id="loadAllInfosNoRelease" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
- select
- <include refid="baseSqlV2">
- <property name="alias" value="instance"/>
- </include>
- from t_ds_task_instance instance
- left join t_ds_task_group_queue que on instance.id = que.task_id
- where instance.process_instance_id = #{processInstanceId}
- and que.status = #{status}
- </select>
<select id="findLastTaskInstances" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
select
<include refid="baseSqlV2">
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java
index 5d454c2206..102110c18f 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapperTest.java
@@ -197,30 +197,6 @@ public class ProcessInstanceMapperTest extends BaseDaoTest {
processInstanceMapper.deleteById(processInstance.getId());
}
- /**
- * test set failover by host and state
- */
- @Test
- public void testSetFailoverByHostAndStateArray() {
-
- int[] stateArray = new int[]{
- WorkflowExecutionStatus.RUNNING_EXECUTION.ordinal(),
- WorkflowExecutionStatus.SUCCESS.ordinal()};
-
- ProcessInstance processInstance = insertOne();
-
- processInstance.setState(WorkflowExecutionStatus.RUNNING_EXECUTION);
- processInstance.setHost("192.168.2.220");
- processInstanceMapper.updateById(processInstance);
- String host = processInstance.getHost();
- int update = processInstanceMapper.setFailoverByHostAndStateArray(host, stateArray);
- Assertions.assertNotEquals(0, update);
-
- processInstance = processInstanceMapper.selectById(processInstance.getId());
- Assertions.assertNull(processInstance.getHost());
- processInstanceMapper.deleteById(processInstance.getId());
- }
-
/**
* test update process instance by state
*/
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java
index 03a9e69ff6..d48dde831a 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapperTest.java
@@ -143,25 +143,6 @@ public class TaskInstanceMapperTest extends BaseDaoTest {
Assertions.assertNotEquals(0, taskInstances.size());
}
- /**
- * test query task instance by process instance id and state
- */
- @Test
- public void testQueryTaskByProcessIdAndState() {
- // insert ProcessInstance
- ProcessInstance processInstance = insertProcessInstance();
-
- // insert taskInstance
- TaskInstance task = insertTaskInstance(processInstance.getId());
- task.setProcessInstanceId(processInstance.getId());
- taskInstanceMapper.updateById(task);
- List<Integer> taskInstances = taskInstanceMapper.queryTaskByProcessIdAndState(
- task.getProcessInstanceId(),
- TaskExecutionStatus.RUNNING_EXECUTION.getCode());
- taskInstanceMapper.deleteById(task.getId());
- Assertions.assertNotEquals(0, taskInstances.size());
- }
-
/**
* test find valid task list by process instance id
*/
@@ -194,66 +175,6 @@ public class TaskInstanceMapperTest extends BaseDaoTest {
Assertions.assertNotEquals(0, taskInstances1.size());
}
- /**
- * test query by host and status
- */
- @Test
- public void testQueryByHostAndStatus() {
- // insert ProcessInstance
- ProcessInstance processInstance = insertProcessInstance();
-
- // insert taskInstance
- TaskInstance task = insertTaskInstance(processInstance.getId());
- task.setHost("111.111.11.11");
- taskInstanceMapper.updateById(task);
-
- List<TaskInstance> taskInstances = taskInstanceMapper.queryByHostAndStatus(
- task.getHost(), new int[]{TaskExecutionStatus.RUNNING_EXECUTION.getCode()});
- taskInstanceMapper.deleteById(task.getId());
- Assertions.assertNotEquals(0, taskInstances.size());
- }
-
- /**
- * test set failover by host and state array
- */
- @Test
- public void testSetFailoverByHostAndStateArray() {
- // insert ProcessInstance
- ProcessInstance processInstance = insertProcessInstance();
-
- // insert taskInstance
- TaskInstance task = insertTaskInstance(processInstance.getId());
- task.setHost("111.111.11.11");
- taskInstanceMapper.updateById(task);
-
- int setResult = taskInstanceMapper.setFailoverByHostAndStateArray(
- task.getHost(),
- new int[]{TaskExecutionStatus.RUNNING_EXECUTION.getCode()},
- TaskExecutionStatus.NEED_FAULT_TOLERANCE);
- taskInstanceMapper.deleteById(task.getId());
- Assertions.assertNotEquals(0, setResult);
- }
-
- /**
- * test query by task instance id and name
- */
- @Test
- public void testQueryByInstanceIdAndName() {
- // insert ProcessInstance
- ProcessInstance processInstance = insertProcessInstance();
-
- // insert taskInstance
- TaskInstance task = insertTaskInstance(processInstance.getId());
- task.setHost("111.111.11.11");
- taskInstanceMapper.updateById(task);
-
- TaskInstance taskInstance = taskInstanceMapper.queryByInstanceIdAndName(
- task.getProcessInstanceId(),
- task.getName());
- taskInstanceMapper.deleteById(task.getId());
- Assertions.assertNotEquals(null, taskInstance);
- }
-
/**
* test query by task instance id and code
*/
@@ -294,37 +215,6 @@ public class TaskInstanceMapperTest extends BaseDaoTest {
Assertions.assertEquals(1, taskInstances.size());
}
- /**
- * test count task instance
- */
- @Test
- public void testCountTask() {
- // insert ProcessInstance
- ProcessInstance processInstance = insertProcessInstance();
-
- // insert taskInstance
- TaskInstance task = insertTaskInstance(processInstance.getId());
- ProcessDefinition definition = new ProcessDefinition();
- definition.setCode(1L);
- definition.setProjectCode(1111L);
- definition.setCreateTime(new Date());
- definition.setUpdateTime(new Date());
- processDefinitionMapper.insert(definition);
- taskInstanceMapper.updateById(task);
-
- int countTask = taskInstanceMapper.countTask(
- new Long[0],
- new int[0]);
- int countTask2 = taskInstanceMapper.countTask(
- new Long[]{definition.getProjectCode()},
- new int[]{task.getId()});
- taskInstanceMapper.deleteById(task.getId());
- processDefinitionMapper.deleteById(definition.getId());
- Assertions.assertEquals(0, countTask);
- Assertions.assertEquals(0, countTask2);
-
- }
-
/**
* test count task instance state by user
*/
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index ba4def7e06..101350e90d 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -45,7 +45,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.service.exceptions.CronParseException;
import org.apache.dolphinscheduler.service.model.TaskNode;
@@ -98,8 +97,6 @@ public interface ProcessService {
void updateTaskDefinitionResources(TaskDefinition taskDefinition);
- List<Integer> findTaskIdByInstanceState(int instanceId, TaskExecutionStatus state);
-
int deleteWorkProcessMapByParentId(int parentWorkProcessId);
ProcessInstance findSubProcessInstance(Integer parentProcessId, Integer parentTaskId);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 130a1c0993..1f27857513 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -763,22 +763,27 @@ public class ProcessServiceImpl implements ProcessService {
case DYNAMIC_GENERATION:
break;
case START_FAILURE_TASK_PROCESS:
- // find failed tasks and init these tasks
- List<Integer> failedList =
- this.findTaskIdByInstanceState(processInstance.getId(), TaskExecutionStatus.FAILURE);
- List<Integer> toleranceList = this.findTaskIdByInstanceState(processInstance.getId(),
- TaskExecutionStatus.NEED_FAULT_TOLERANCE);
- List<Integer> killedList =
- this.findTaskIdByInstanceState(processInstance.getId(), TaskExecutionStatus.KILL);
- cmdParam.remove(CommandKeyConstants.CMD_PARAM_RECOVERY_START_NODE_STRING);
-
- failedList.addAll(killedList);
- failedList.addAll(toleranceList);
- for (Integer taskId : failedList) {
- initTaskInstance(taskInstanceDao.queryById(taskId));
+ case RECOVER_SUSPENDED_PROCESS:
+ List<TaskInstance> needToStartTaskInstances = taskInstanceDao
+ .queryValidTaskListByWorkflowInstanceId(processInstance.getId(), processInstance.getTestFlag())
+ .stream()
+ .filter(taskInstance -> {
+ TaskExecutionStatus state = taskInstance.getState();
+ return state == TaskExecutionStatus.FAILURE
+ || state == TaskExecutionStatus.PAUSE
+ || state == TaskExecutionStatus.NEED_FAULT_TOLERANCE
+ || state == TaskExecutionStatus.KILL;
+ })
+ .collect(Collectors.toList());
+
+ for (TaskInstance taskInstance : needToStartTaskInstances) {
+ initTaskInstance(taskInstance);
}
- cmdParam.put(CommandKeyConstants.CMD_PARAM_RECOVERY_START_NODE_STRING,
- String.join(Constants.COMMA, convertIntListToString(failedList)));
+ String startTaskInstanceIds = needToStartTaskInstances.stream()
+ .map(TaskInstance::getId)
+ .map(String::valueOf)
+ .collect(Collectors.joining(Constants.COMMA));
+ cmdParam.put(CommandKeyConstants.CMD_PARAM_RECOVERY_START_NODE_STRING, startTaskInstanceIds);
processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam));
processInstance.setRunTimes(runTime + 1);
break;
@@ -786,20 +791,6 @@ public class ProcessServiceImpl implements ProcessService {
break;
case RECOVER_WAITING_THREAD:
break;
- case RECOVER_SUSPENDED_PROCESS:
- // find pause tasks and init task's state
- cmdParam.remove(CommandKeyConstants.CMD_PARAM_RECOVERY_START_NODE_STRING);
- List<Integer> stopNodeList = findTaskIdByInstanceState(processInstance.getId(),
- TaskExecutionStatus.KILL);
- for (Integer taskId : stopNodeList) {
- // initialize the pause state
- initTaskInstance(taskInstanceDao.queryById(taskId));
- }
- cmdParam.put(CommandKeyConstants.CMD_PARAM_RECOVERY_START_NODE_STRING,
- String.join(Constants.COMMA, convertIntListToString(stopNodeList)));
- processInstance.setCommandParam(JSONUtils.toJsonString(cmdParam));
- processInstance.setRunTimes(runTime + 1);
- break;
case RECOVER_TOLERANCE_FAULT_PROCESS:
// recover tolerance fault process
processInstance.setRecovery(Flag.YES);
@@ -1312,18 +1303,6 @@ public class ProcessServiceImpl implements ProcessService {
return resourceInfo;
}
- /**
- * get id list by task state
- *
- * @param instanceId instanceId
- * @param state state
- * @return task instance states
- */
- @Override
- public List<Integer> findTaskIdByInstanceState(int instanceId, TaskExecutionStatus state) {
- return taskInstanceMapper.queryTaskByProcessIdAndState(instanceId, state.getCode());
- }
-
/**
* delete work process map by parent process id
*
(dolphinscheduler) 10/18: [Bug][force-success] force success add end time (#15144)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 7d660ec2f93c69599ab9620c9456be597894f6dc
Author: 小可耐 <46...@users.noreply.github.com>
AuthorDate: Mon Feb 5 14:23:16 2024 +0800
[Bug][force-success] force success add end time (#15144)
Co-authored-by: Rick Cheng <ri...@gmail.com>
(cherry picked from commit e5a208f363abcb31bb7e8ffee16c7a142671c667)
---
.../dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
index d3426dfa3e..f06f8115a9 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
@@ -255,6 +255,7 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst
// change the state of the task instance
task.setState(TaskExecutionStatus.FORCED_SUCCESS);
+ task.setEndTime(new Date());
int changedNum = taskInstanceMapper.updateById(task);
if (changedNum > 0) {
processService.forceProcessInstanceSuccessByTaskInstanceId(taskInstanceId);
(dolphinscheduler) 12/18: fix: data quality may fail in docker mode (#15563)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit e2d08fc27d99cb8ac6393ccd2c0552cff0895d99
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Mon Feb 5 17:14:59 2024 +0800
fix: data quality may fail in docker mode (#15563)
(cherry picked from commit 91d56f48601279eeb7de1a057e5c4425f86f2198)
---
deploy/kubernetes/dolphinscheduler/README.md | 2 +-
deploy/kubernetes/dolphinscheduler/values.yaml | 2 +-
docs/docs/en/architecture/configuration.md | 2 +-
docs/docs/en/guide/data-quality.md | 2 +-
docs/docs/en/guide/resource/configuration.md | 7 +++---
docs/docs/en/guide/upgrade/incompatible.md | 1 +
docs/docs/zh/architecture/configuration.md | 2 +-
docs/docs/zh/guide/data-quality.md | 2 +-
docs/docs/zh/guide/resource/configuration.md | 7 +++---
.../resources/docker/file-manage/common.properties | 7 +++---
.../src/main/resources/common.properties | 7 +++---
.../src/test/resources/common.properties | 7 +++---
.../plugin/datasource/api/utils/CommonUtils.java | 29 ++++++++++++++++------
.../resources/docker/file-manage/common.properties | 7 +++---
.../src/main/docker/Dockerfile | 1 +
.../plugin/task/api/TaskConstants.java | 4 +--
.../src/test/resources/common.properties | 7 +++---
17 files changed, 60 insertions(+), 36 deletions(-)
diff --git a/deploy/kubernetes/dolphinscheduler/README.md b/deploy/kubernetes/dolphinscheduler/README.md
index e92be96011..992267e5e4 100644
--- a/deploy/kubernetes/dolphinscheduler/README.md
+++ b/deploy/kubernetes/dolphinscheduler/README.md
@@ -121,7 +121,7 @@ Please refer to the [Quick Start in Kubernetes](../../../docs/docs/en/guide/inst
| conf.common."alert.rpc.port" | int | `50052` | rpc port |
| conf.common."appId.collect" | string | `"log"` | way to collect applicationId: log, aop |
| conf.common."conda.path" | string | `"/opt/anaconda3/etc/profile.d/conda.sh"` | set path of conda.sh |
-| conf.common."data-quality.jar.name" | string | `"dolphinscheduler-data-quality-dev-SNAPSHOT.jar"` | data quality option |
+| conf.common."data-quality.jar.dir" | string | `nil` | data quality option |
| conf.common."data.basedir.path" | string | `"/tmp/dolphinscheduler"` | user data local directory path, please make sure the directory exists and have read write permissions |
| conf.common."datasource.encryption.enable" | bool | `false` | datasource encryption enable |
| conf.common."datasource.encryption.salt" | string | `"!@#$%^&*"` | datasource encryption salt |
diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml
index ca4b5b51a7..7370015980 100644
--- a/deploy/kubernetes/dolphinscheduler/values.yaml
+++ b/deploy/kubernetes/dolphinscheduler/values.yaml
@@ -328,7 +328,7 @@ conf:
datasource.encryption.salt: '!@#$%^&*'
# -- data quality option
- data-quality.jar.name: dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+ data-quality.jar.dir:
# -- Whether hive SQL is executed in the same session
support.hive.oneSession: false
diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md
index f4ab1435d5..b9a26b865c 100644
--- a/docs/docs/en/architecture/configuration.md
+++ b/docs/docs/en/architecture/configuration.md
@@ -226,7 +226,7 @@ The default configuration is as follows:
| yarn.job.history.status.address | http://ds1:19888/ws/v1/history/mapreduce/jobs/%s | job history status url of yarn [...]
| datasource.encryption.enable | false | whether to enable datasource encryption [...]
| datasource.encryption.salt | !@#$%^&* | the salt of the datasource encryption [...]
-| data-quality.jar.name | dolphinscheduler-data-quality-dev-SNAPSHOT.jar | the jar of data quality [...]
+| data-quality.jar.dir | | the jar of data quality [...]
| support.hive.oneSession | false | specify whether hive SQL is executed in the same session [...]
| sudo.enable | true | whether to enable sudo [...]
| alert.rpc.port | 50052 | the RPC port of Alert Server [...]
diff --git a/docs/docs/en/guide/data-quality.md b/docs/docs/en/guide/data-quality.md
index f6aa7a06b2..dca777d76f 100644
--- a/docs/docs/en/guide/data-quality.md
+++ b/docs/docs/en/guide/data-quality.md
@@ -12,7 +12,7 @@ The execution logic of the data quality task is as follows:
- The current data quality task result is stored in the `t_ds_dq_execute_result` table of `dolphinscheduler`
`Worker` sends the task result to `Master`, after `Master` receives `TaskResponse`, it will judge whether the task type is `DataQualityTask`, if so, it will read the corresponding result from `t_ds_dq_execute_result` according to `taskInstanceId`, and then The result is judged according to the check mode, operator and threshold configured by the user.
- If the result is a failure, the corresponding operation, alarm or interruption will be performed according to the failure policy configured by the user.
-- If you package `data-quality` separately, remember to modify the package name to be consistent with `data-quality.jar.name` in `common.properties` with attribute name `data-quality.jar.name`
+- If you package `data-quality` separately, remember to modify the package name to be consistent with `data-quality.jar.dir` in `common.properties` with attribute name `data-quality.jar.dir`
- If the old version is upgraded and used, you need to execute the `sql` update script to initialize the database before running.
- `dolphinscheduler-data-quality-dev-SNAPSHOT.jar` was built with no dependencies. If a `JDBC` driver is required, you can set the `-jars` parameter in the `node settings` `Option Parameters`, e.g. `--jars /lib/jars/mysql-connector-java-8.0.16.jar`.
- Currently only `MySQL`, `PostgreSQL` and `HIVE` data sources have been tested, other data sources have not been tested yet.
diff --git a/docs/docs/en/guide/resource/configuration.md b/docs/docs/en/guide/resource/configuration.md
index 42e1925e89..67c68a22c2 100644
--- a/docs/docs/en/guide/resource/configuration.md
+++ b/docs/docs/en/guide/resource/configuration.md
@@ -152,9 +152,10 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
-# if you re-build it alone, or auto discovery mechanism fail
-data-quality.jar.name=
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md
index 4580a7d13d..f45af712c3 100644
--- a/docs/docs/en/guide/upgrade/incompatible.md
+++ b/docs/docs/en/guide/upgrade/incompatible.md
@@ -11,6 +11,7 @@ This document records the incompatible updates between each version. You need to
* Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
* Remove `deleteSource` in `download()` of `StorageOperate` ([#14084](https://github.com/apache/dolphinscheduler/pull/14084))
* Remove default key for attribute `data-quality.jar.name` in `common.properties` ([#15551](https://github.com/apache/dolphinscheduler/pull/15551))
+* Rename attribute `data-quality.jar.name` to `data-quality.jar.dir` in `common.properties` and represent for directory ([#15563](https://github.com/apache/dolphinscheduler/pull/15563))
## 3.2.0
diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md
index 65113b76a1..0b3ea9bc5b 100644
--- a/docs/docs/zh/architecture/configuration.md
+++ b/docs/docs/zh/architecture/configuration.md
@@ -226,7 +226,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn/applicationId
| yarn.job.history.status.address | http://ds1:19888/ws/v1/history/mapreduce/jobs/%s | yarn的作业历史状态URL |
| datasource.encryption.enable | false | 是否启用datasource 加密 |
| datasource.encryption.salt | !@#$%^&* | datasource加密使用的salt |
-| data-quality.jar.name | dolphinscheduler-data-quality-dev-SNAPSHOT.jar | 配置数据质量使用的jar包 |
+| data-quality.jar.dir | | 配置数据质量使用的jar包 |
| support.hive.oneSession | false | 设置hive SQL是否在同一个session中执行 |
| sudo.enable | true | 是否开启sudo |
| alert.rpc.port | 50052 | Alert Server的RPC端口 |
diff --git a/docs/docs/zh/guide/data-quality.md b/docs/docs/zh/guide/data-quality.md
index 2a098a3216..17b2a55cb2 100644
--- a/docs/docs/zh/guide/data-quality.md
+++ b/docs/docs/zh/guide/data-quality.md
@@ -13,7 +13,7 @@
>
## 注意事项
-- 如果单独打包`data-quality`的话，记得修改包名和`data-quality.jar.name`一致，配置内容在 `common.properties` 中的 `data-quality.jar.name`
+- 如果单独打包`data-quality`的话，记得修改包路径和`data-quality.jar.dir`一致，配置内容在 `common.properties` 中的 `data-quality.jar.dir`
- 如果是老版本升级使用，运行之前需要先执行`SQL`更新脚本进行数据库初始化。
- 当前 `dolphinscheduler-data-quality-dev-SNAPSHOT.jar` 是瘦包，不包含任何 `JDBC` 驱动。
如果有 `JDBC` 驱动需要，可以在`节点设置` `选项参数`处设置 `--jars` 参数，
diff --git a/docs/docs/zh/guide/resource/configuration.md b/docs/docs/zh/guide/resource/configuration.md
index 57d0935e09..739d6fb30c 100644
--- a/docs/docs/zh/guide/resource/configuration.md
+++ b/docs/docs/zh/guide/resource/configuration.md
@@ -156,9 +156,10 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
-# if you re-build it alone, or auto discovery mechanism fail
-data-quality.jar.name=
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties
index d43e55e822..96879cc272 100644
--- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties
+++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties
@@ -84,9 +84,10 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
-# if you re-build it alone, or auto discovery mechanism fail
-data-quality.jar.name=
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
index 451a0f734c..669d3dfef3 100644
--- a/dolphinscheduler-common/src/main/resources/common.properties
+++ b/dolphinscheduler-common/src/main/resources/common.properties
@@ -120,9 +120,10 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
-# if you re-build it alone, or auto discovery mechanism fail
-data-quality.jar.name=
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-common/src/test/resources/common.properties b/dolphinscheduler-common/src/test/resources/common.properties
index 107977df7f..7f66a32a23 100644
--- a/dolphinscheduler-common/src/test/resources/common.properties
+++ b/dolphinscheduler-common/src/test/resources/common.properties
@@ -115,9 +115,10 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
-# if you re-build it alone, or auto discovery mechanism fail
-data-quality.jar.name=
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
index a4e64594c0..1c24785c2f 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
@@ -18,7 +18,7 @@
package org.apache.dolphinscheduler.plugin.datasource.api.utils;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_STORAGE_TYPE;
-import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATA_QUALITY_JAR_NAME;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DATA_QUALITY_JAR_DIR;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.JAVA_SECURITY_KRB5_CONF;
@@ -133,14 +133,28 @@ public class CommonUtils {
}
public static String getDataQualityJarPath() {
- String dqsJarPath = PropertyUtils.getString(DATA_QUALITY_JAR_NAME);
+ log.info("Trying to get data quality jar in path");
+ String dqJarDir = PropertyUtils.getString(DATA_QUALITY_JAR_DIR);
+
+ if (StringUtils.isNotEmpty(dqJarDir)) {
+ log.info(
+ "Configuration data-quality.jar.dir is not empty, will try to get data quality jar from directory {}",
+ dqJarDir);
+ getDataQualityJarPathFromPath(dqJarDir).ifPresent(jarName -> DEFAULT_DATA_QUALITY_JAR_PATH = jarName);
+ }
+
+ if (StringUtils.isEmpty(DEFAULT_DATA_QUALITY_JAR_PATH)) {
+ log.info("data quality jar path is empty, will try to auto discover it from build-in rules.");
+ getDefaultDataQualityJarPath();
+ }
- if (StringUtils.isEmpty(dqsJarPath)) {
- log.info("data quality jar path is empty, will try to get it from data quality jar name");
- return getDefaultDataQualityJarPath();
+ if (StringUtils.isEmpty(DEFAULT_DATA_QUALITY_JAR_PATH)) {
+ log.error(
+ "Can not find data quality jar in both configuration and auto discover, please check your configuration or report a bug.");
+ throw new RuntimeException("data quality jar path is empty");
}
- return dqsJarPath;
+ return DEFAULT_DATA_QUALITY_JAR_PATH;
}
private static String getDefaultDataQualityJarPath() {
@@ -173,7 +187,7 @@ public class CommonUtils {
log.info("Try to get data quality jar from path {}", path);
File[] jars = new File(path).listFiles();
if (jars == null) {
- log.warn("No data quality related jar found from path {}", path);
+ log.warn("No any files find given path {}", path);
return Optional.empty();
}
for (File jar : jars) {
@@ -181,6 +195,7 @@ public class CommonUtils {
return Optional.of(jar.getAbsolutePath());
}
}
+ log.warn("No data quality related jar found from path {}", path);
return Optional.empty();
}
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
index b5f61011b3..7583b3293a 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
@@ -95,9 +95,10 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option, it would auto discovery from libs directory. You can also specific the jar name in libs directory
-# if you re-build it alone, or auto discovery mechanism fail
-data-quality.jar.name=
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-standalone-server/src/main/docker/Dockerfile b/dolphinscheduler-standalone-server/src/main/docker/Dockerfile
index 40158ff8e1..6eebf04162 100644
--- a/dolphinscheduler-standalone-server/src/main/docker/Dockerfile
+++ b/dolphinscheduler-standalone-server/src/main/docker/Dockerfile
@@ -20,6 +20,7 @@ FROM eclipse-temurin:8-jdk
ENV DOCKER true
ENV TZ Asia/Shanghai
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
+ENV DATA_QUALITY_JAR_DIR /opt/dolphinscheduler/libs/worker-server
RUN apt update ; \
apt install -y sudo ; \
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java
index fe8d9a77bf..43734416e7 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java
@@ -358,9 +358,9 @@ public class TaskConstants {
public static final String RESOURCE_UPLOAD_PATH = "resource.storage.upload.base.path";
/**
- * data.quality.jar.name
+ * data.quality.jar.dir
*/
- public static final String DATA_QUALITY_JAR_NAME = "data-quality.jar.name";
+ public static final String DATA_QUALITY_JAR_DIR = "data-quality.jar.dir";
public static final String TASK_TYPE_CONDITIONS = "CONDITIONS";
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties
index 9855d855e9..402112263f 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties
@@ -84,9 +84,10 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
-# if you re-build it alone, or auto discovery mechanism fail
-data-quality.jar.name=
+# data quality jar directory path, it would auto discovery data quality jar from this given dir. You should keep it empty if you do not change anything in
+# data-quality, it will auto discovery by dolphinscheduler itself. Change it only if you want to use your own data-quality jar and it is not in worker-server
+# libs directory(but may sure your jar name start with `dolphinscheduler-data-quality`).
+data-quality.jar.dir=
#data-quality.error.output.path=/tmp/data-quality-error-data
(dolphinscheduler) 08/18: [Bug][Task Api] fix 'MACPATTERN' in ProcessUtils and cover all cases on MacOS in ProcessUtilsTest (#15480) (#15529)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 19adf25920305fab3cf68ab498e12b793228176b
Author: dada Sun <70...@qq.com>
AuthorDate: Mon Feb 5 11:41:21 2024 +0800
[Bug][Task Api] fix 'MACPATTERN' in ProcessUtils and cover all cases on MacOS in ProcessUtilsTest (#15480) (#15529)
Co-authored-by: Rick Cheng <ri...@gmail.com>
(cherry picked from commit 4d6af516b9bbc1587318f9a305d82b461c124a30)
---
.../apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java | 2 +-
.../dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java
index 46b8ff4c7b..7b61a1eaec 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtils.java
@@ -71,7 +71,7 @@ public final class ProcessUtils {
* Initialization regularization, solve the problem of pre-compilation performance,
* avoid the thread safety problem of multi-thread operation
*/
- private static final Pattern MACPATTERN = Pattern.compile("-[+|-]-\\s(\\d+)");
+ private static final Pattern MACPATTERN = Pattern.compile("-[+|-][-|=]\\s(\\d+)");
/**
* Expression of PID recognition in Windows scene
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java
index ef10e73325..14caf470d7 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/ProcessUtilsTest.java
@@ -38,6 +38,7 @@ public class ProcessUtilsTest {
String command;
MockedStatic<OSUtils> osUtilsMockedStatic = Mockito.mockStatic(OSUtils.class);
if (SystemUtils.IS_OS_MAC) {
+ pids = "-+= 6279 sudo -+- 6282 558_1497.sh --- 6354 sleep";
command = String.format("%s -sp %d", TaskConstants.PSTREE, processId);
} else if (SystemUtils.IS_OS_LINUX) {
command = String.format("%s -p %d", TaskConstants.PSTREE, processId);
@@ -54,6 +55,7 @@ public class ProcessUtilsTest {
String exceptPidsStr2 = "2000 2100 2101";
String command2;
if (SystemUtils.IS_OS_MAC) {
+ pids2 = "-+= 2000 apache2 -+- 2100 222332-apache2-submit_task.py --- 2101 apache2";
command2 = String.format("%s -sp %d", TaskConstants.PSTREE, processId2);
} else if (SystemUtils.IS_OS_LINUX) {
command2 = String.format("%s -p %d", TaskConstants.PSTREE, processId2);
@@ -70,6 +72,7 @@ public class ProcessUtilsTest {
String exceptPidsStr3 = "5000 6000 7000 7100";
String command3;
if (SystemUtils.IS_OS_MAC) {
+ pids3 = "-+= 5000 sshd -+- 6000 sshd --= 7000 bash --- 7100 python";
command3 = String.format("%s -sp %d", TaskConstants.PSTREE, processId3);
} else if (SystemUtils.IS_OS_LINUX) {
command3 = String.format("%s -p %d", TaskConstants.PSTREE, processId3);
(dolphinscheduler) 07/18: fix switch js (#15487)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit ca93aa8e260b4e227c054d42aca20b272574e851
Author: caishunfeng <ca...@gmail.com>
AuthorDate: Mon Feb 5 10:23:47 2024 +0800
fix switch js (#15487)
Co-authored-by: Rick Cheng <ri...@gmail.com>
Co-authored-by: Eric Gao <er...@gmail.com>
(cherry picked from commit ef9ed3db55cb1647886b06c2b2c6a5cfcdccfb5c)
---
.../server/master/utils/SwitchTaskUtils.java | 17 +++++++++++++++++
.../server/master/utils/SwitchTaskUtilsTest.java | 14 ++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java
index 4e1c303138..f4ebd0c60c 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtils.java
@@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
import org.apache.commons.collections4.MapUtils;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -33,6 +34,7 @@ import javax.script.ScriptException;
import lombok.extern.slf4j.Slf4j;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
@Slf4j
public class SwitchTaskUtils {
@@ -41,6 +43,15 @@ public class SwitchTaskUtils {
private static final ScriptEngine engine;
private static final String rgex = "['\"]*\\$\\{(.*?)\\}['\"]*";
+ private static final Set<String> blackKeySet = Sets.newHashSet(
+ "java",
+ "invoke",
+ "new",
+ "eval",
+ "function",
+ "import",
+ "\\\\");
+
static {
manager = new ScriptEngineManager();
engine = manager.getEngineByName("js");
@@ -83,6 +94,12 @@ public class SwitchTaskUtils {
content = content.replace("${" + paramName + "}", value);
}
+ for (String blackKey : blackKeySet) {
+ if (content.contains(blackKey)) {
+ throw new IllegalArgumentException("condition is not valid, please check it. condition: " + condition);
+ }
+ }
+
// if not replace any params, throw exception to avoid illegal condition
if (originContent.equals(content)) {
throw new IllegalArgumentException("condition is not valid, please check it. condition: " + condition);
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java
index 044e916f56..18e97f8e80 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/utils/SwitchTaskUtilsTest.java
@@ -52,5 +52,19 @@ public class SwitchTaskUtilsTest {
Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
SwitchTaskUtils.generateContentWithTaskParams(content, globalParams, varParams);
});
+
+ String cmd = "bash /tmp/shell";
+ String cmdContent = "java.lang.Runtime.getRuntime().exec(\"${cmd}\")";
+ globalParams.put("cmd", new Property("cmd", Direct.IN, DataType.VARCHAR, cmd));
+ Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
+ SwitchTaskUtils.generateContentWithTaskParams(cmdContent, globalParams, varParams);
+ });
+
+ String contentWithUnicode =
+ "\\\\u006a\\\\u0061\\\\u0076\\\\u0061\\\\u002e\\\\u006c\\\\u0061\\\\u006e\\\\u0067\\\\u002e\\\\u0052\\\\u0075\\\\u006e\\\\u0074\\\\u0069\\\\u006d\\\\u0065.getRuntime().exec(\\\"open -a Calculator.app\\";
+ Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
+ SwitchTaskUtils.generateContentWithTaskParams(contentWithUnicode, globalParams, varParams);
+ });
+
}
}
(dolphinscheduler) 18/18: [fix][worker]:Fix the issue of missing disk usage reporting information in the worker (#15565)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 58aba58079de7bfb68d13db9bb61b5bfcc9e2045
Author: seckiller <45...@qq.com>
AuthorDate: Tue Feb 6 16:43:29 2024 +0800
[fix][worker]:Fix the issue of missing disk usage reporting information in the worker (#15565)
Co-authored-by: Rick Cheng <ri...@gmail.com>
(cherry picked from commit 89da67ddebb0d07f9106f226ae810bb44faa71d4)
---
.../apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java
index 7a4535c79f..57349e1448 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/task/WorkerHeartBeatTask.java
@@ -68,6 +68,7 @@ public class WorkerHeartBeatTask extends BaseHeartBeatTask<WorkerHeartBeat> {
.cpuUsage(systemMetrics.getTotalCpuUsedPercentage())
.jvmMemoryUsage(systemMetrics.getJvmMemoryUsedPercentage())
.memoryUsage(systemMetrics.getSystemMemoryUsedPercentage())
+ .diskUsage(systemMetrics.getDiskUsedPercentage())
.processId(processId)
.workerHostWeight(workerConfig.getHostWeight())
.threadPoolUsage(workerTaskExecutorThreadPool.getRunningTaskExecutorSize()
(dolphinscheduler) 14/18: [Improvement][UT] Improve Worker registry coverage (#15380)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 8e9537ad33e05a9e141fe5121c620af723528ba5
Author: John Huang <pe...@gmail.com>
AuthorDate: Tue Feb 6 10:45:53 2024 +0800
[Improvement][UT] Improve Worker registry coverage (#15380)
Co-authored-by: fuchanghai <ch...@apache.org>
Co-authored-by: Eric Gao <er...@gmail.com>
Co-authored-by: Rick Cheng <ri...@gmail.com>
(cherry picked from commit 69676b445c6a87dbd97c8bfd5797a26600e1bef6)
---
.../server/worker/registry/WorkerStopStrategy.java | 3 -
.../worker/registry/WorkerWaitingStrategy.java | 11 ++
.../WorkerConnectionStateListenerTest.java | 60 +++++++
.../worker/registry/WorkerRegistryClientTest.java | 40 +++--
.../server/worker/registry/WorkerStrategyTest.java | 187 +++++++++++++++++++++
5 files changed, 284 insertions(+), 17 deletions(-)
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java
index ee4e960a80..4c575a29e8 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStopStrategy.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.server.worker.registry;
import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.registry.api.StrategyType;
-import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
import lombok.extern.slf4j.Slf4j;
@@ -34,8 +33,6 @@ public class WorkerStopStrategy implements WorkerConnectStrategy {
@Autowired
public RegistryClient registryClient;
- @Autowired
- private WorkerConfig workerConfig;
@Override
public void disconnect() {
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java
index 4437e3545d..2be156b5df 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerWaitingStrategy.java
@@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecutorThread
import java.time.Duration;
+import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -52,6 +53,16 @@ public class WorkerWaitingStrategy implements WorkerConnectStrategy {
@Autowired
private WorkerTaskExecutorThreadPool workerManagerThread;
+ public WorkerWaitingStrategy(@NonNull WorkerConfig workerConfig,
+ @NonNull RegistryClient registryClient,
+ @NonNull MessageRetryRunner messageRetryRunner,
+ @NonNull WorkerTaskExecutorThreadPool workerManagerThread) {
+ this.workerConfig = workerConfig;
+ this.registryClient = registryClient;
+ this.messageRetryRunner = messageRetryRunner;
+ this.workerManagerThread = workerManagerThread;
+ }
+
@Override
public void disconnect() {
try {
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListenerTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListenerTest.java
new file mode 100644
index 0000000000..14d75779b9
--- /dev/null
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerConnectionStateListenerTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.registry;
+
+import static org.mockito.Mockito.times;
+
+import org.apache.dolphinscheduler.registry.api.ConnectionState;
+import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * worker registry test
+ */
+@ExtendWith(MockitoExtension.class)
+public class WorkerConnectionStateListenerTest {
+
+ private static final Logger log = LoggerFactory.getLogger(WorkerConnectionStateListenerTest.class);
+ @InjectMocks
+ private WorkerConnectionStateListener workerConnectionStateListener;
+ @Mock
+ private WorkerConfig workerConfig;
+ @Mock
+ private WorkerConnectStrategy workerConnectStrategy;
+
+ @Test
+ public void testWorkerConnectionStateListener() {
+ workerConnectionStateListener.onUpdate(ConnectionState.CONNECTED);
+
+ workerConnectionStateListener.onUpdate(ConnectionState.RECONNECTED);
+ Mockito.verify(workerConnectStrategy, times(1)).reconnect();
+
+ workerConnectionStateListener.onUpdate(ConnectionState.SUSPENDED);
+
+ workerConnectionStateListener.onUpdate(ConnectionState.DISCONNECTED);
+ Mockito.verify(workerConnectStrategy, times(1)).disconnect();
+ }
+}
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
index f44c759420..8c7071cf60 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClientTest.java
@@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.server.worker.registry;
import static org.mockito.BDDMockito.given;
+import org.apache.dolphinscheduler.common.IStoppable;
+import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.meter.metrics.MetricsProvider;
import org.apache.dolphinscheduler.meter.metrics.SystemMetrics;
@@ -29,6 +31,9 @@ import org.apache.dolphinscheduler.server.worker.config.WorkerServerLoadProtecti
import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecutorThreadPool;
import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Optional;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -37,6 +42,8 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* worker registry test
@@ -44,26 +51,24 @@ import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
public class WorkerRegistryClientTest {
+ private static final Logger log = LoggerFactory.getLogger(WorkerRegistryClientTest.class);
@InjectMocks
private WorkerRegistryClient workerRegistryClient;
-
@Mock
private RegistryClient registryClient;
-
@Mock
private WorkerConfig workerConfig;
-
@Mock
private MetricsProvider metricsProvider;
-
@Mock
private WorkerTaskExecutorThreadPool workerManagerThread;
-
@Mock
private WorkerConnectStrategy workerConnectStrategy;
+ @Mock
+ private IStoppable stoppable;
@Test
- public void testStart() {
+ public void testWorkerRegistryClientbasic() {
given(workerConfig.getWorkerAddress()).willReturn(NetUtils.getAddr(1234));
given(workerConfig.getMaxHeartbeatInterval()).willReturn(Duration.ofSeconds(1));
@@ -75,16 +80,23 @@ public class WorkerRegistryClientTest {
workerRegistryClient.initWorkRegistry();
workerRegistryClient.start();
- Assertions.assertTrue(true);
+ workerRegistryClient.setRegistryStoppable(stoppable);
}
@Test
- public void testUnRegistry() {
-
- }
-
- @Test
- public void testGetWorkerZkPaths() {
-
+ public void testWorkerRegistryClientgetAlertServerAddress() {
+ given(registryClient.getServerList(Mockito.any(RegistryNodeType.class)))
+ .willReturn(new ArrayList<Server>());
+ Assertions.assertEquals(workerRegistryClient.getAlertServerAddress(), Optional.empty());
+ Mockito.reset(registryClient);
+ String host = "test";
+ Integer port = 1;
+ Server server = new Server();
+ server.setHost(host);
+ server.setPort(port);
+ given(registryClient.getServerList(Mockito.any(RegistryNodeType.class)))
+ .willReturn(new ArrayList<Server>(Arrays.asList(server)));
+ Assertions.assertEquals(workerRegistryClient.getAlertServerAddress().get().getAddress(),
+ String.format("%s:%d", host, port));
}
}
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStrategyTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStrategyTest.java
new file mode 100644
index 0000000000..671fac5277
--- /dev/null
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerStrategyTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.worker.registry;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.Mockito.doNothing;
+
+import org.apache.dolphinscheduler.common.IStoppable;
+import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleException;
+import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
+import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties;
+import org.apache.dolphinscheduler.registry.api.RegistryClient;
+import org.apache.dolphinscheduler.registry.api.RegistryException;
+import org.apache.dolphinscheduler.registry.api.StrategyType;
+import org.apache.dolphinscheduler.server.worker.config.WorkerConfig;
+import org.apache.dolphinscheduler.server.worker.message.MessageRetryRunner;
+import org.apache.dolphinscheduler.server.worker.rpc.WorkerRpcServer;
+import org.apache.dolphinscheduler.server.worker.runner.WorkerTaskExecutorThreadPool;
+
+import java.time.Duration;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * worker registry test
+ */
+@ExtendWith(MockitoExtension.class)
+public class WorkerStrategyTest {
+
+ private static final Logger log = LoggerFactory.getLogger(WorkerStrategyTest.class);
+ @Mock
+ private RegistryClient registryClient;
+ @Mock
+ private IStoppable stoppable;
+ @Mock
+ private WorkerConfig workerConfig;
+ @Mock
+ private WorkerRpcServer workerRpcServer;
+ @Mock
+ private MessageRetryRunner messageRetryRunner;
+ @Mock
+ private WorkerTaskExecutorThreadPool workerManagerThread;
+ @Mock
+ private ConnectStrategyProperties connectStrategyProperties;
+
+ @Test
+ public void testWorkerStopStrategy() {
+ given(registryClient.getStoppable())
+ .willReturn(stoppable);
+ WorkerStopStrategy workerStopStrategy = new WorkerStopStrategy();
+ workerStopStrategy.registryClient = registryClient;
+ workerStopStrategy.reconnect();
+ workerStopStrategy.disconnect();
+ Assertions.assertEquals(workerStopStrategy.getStrategyType(), StrategyType.STOP);
+ }
+
+ @Test
+ public void testWorkerWaitingStrategyreconnect() {
+ WorkerWaitingStrategy workerWaitingStrategy = new WorkerWaitingStrategy(
+ workerConfig,
+ registryClient,
+ messageRetryRunner,
+ workerManagerThread);
+ Assertions.assertEquals(workerWaitingStrategy.getStrategyType(), StrategyType.WAITING);
+
+ try (
+ MockedStatic<ServerLifeCycleManager> serverLifeCycleManagerMockedStatic =
+ Mockito.mockStatic(ServerLifeCycleManager.class)) {
+ serverLifeCycleManagerMockedStatic
+ .when(() -> ServerLifeCycleManager.isRunning())
+ .thenReturn(true);
+ workerWaitingStrategy.reconnect();
+
+ }
+
+ try (
+ MockedStatic<ServerLifeCycleManager> serverLifeCycleManagerMockedStatic =
+ Mockito.mockStatic(ServerLifeCycleManager.class)) {
+ doNothing().when(stoppable).stop(anyString());
+ given(registryClient.getStoppable())
+ .willReturn(stoppable);
+ serverLifeCycleManagerMockedStatic
+ .when(() -> ServerLifeCycleManager.recoverFromWaiting())
+ .thenThrow(new ServerLifeCycleException(""));
+ workerWaitingStrategy.reconnect();
+ }
+
+ try (
+ MockedStatic<ServerLifeCycleManager> serverLifeCycleManagerMockedStatic =
+ Mockito.mockStatic(ServerLifeCycleManager.class)) {
+ serverLifeCycleManagerMockedStatic
+ .when(() -> ServerLifeCycleManager.recoverFromWaiting())
+ .thenAnswer(invocation -> null);
+ workerWaitingStrategy.reconnect();
+ }
+ }
+
+ @Test
+ public void testWorkerWaitingStrategydisconnect() {
+ WorkerWaitingStrategy workerWaitingStrategy = new WorkerWaitingStrategy(
+ workerConfig,
+ registryClient,
+ messageRetryRunner,
+ workerManagerThread);
+ Assertions.assertEquals(workerWaitingStrategy.getStrategyType(), StrategyType.WAITING);
+
+ try (
+ MockedStatic<ServerLifeCycleManager> serverLifeCycleManagerMockedStatic =
+ Mockito.mockStatic(ServerLifeCycleManager.class)) {
+ doNothing().when(stoppable).stop(anyString());
+ given(registryClient.getStoppable())
+ .willReturn(stoppable);
+ serverLifeCycleManagerMockedStatic
+ .when(() -> ServerLifeCycleManager.toWaiting())
+ .thenThrow(new ServerLifeCycleException(""));
+ workerWaitingStrategy.disconnect();
+ }
+
+ try (
+ MockedStatic<ServerLifeCycleManager> serverLifeCycleManagerMockedStatic =
+ Mockito.mockStatic(ServerLifeCycleManager.class)) {
+ given(connectStrategyProperties.getMaxWaitingTime()).willReturn(Duration.ofSeconds(1));
+ given(workerConfig.getRegistryDisconnectStrategy()).willReturn(connectStrategyProperties);
+ Mockito.reset(registryClient);
+ doNothing().when(registryClient).connectUntilTimeout(any());
+ serverLifeCycleManagerMockedStatic
+ .when(() -> ServerLifeCycleManager.toWaiting())
+ .thenAnswer(invocation -> null);
+ workerWaitingStrategy.disconnect();
+ }
+
+ try (
+ MockedStatic<ServerLifeCycleManager> serverLifeCycleManagerMockedStatic =
+ Mockito.mockStatic(ServerLifeCycleManager.class)) {
+ given(connectStrategyProperties.getMaxWaitingTime()).willReturn(Duration.ofSeconds(1));
+ given(workerConfig.getRegistryDisconnectStrategy()).willReturn(connectStrategyProperties);
+ Mockito.reset(registryClient);
+ doNothing().when(stoppable).stop(anyString());
+ given(registryClient.getStoppable())
+ .willReturn(stoppable);
+ Mockito.doThrow(new RegistryException("TEST")).when(registryClient).connectUntilTimeout(any());
+ serverLifeCycleManagerMockedStatic
+ .when(() -> ServerLifeCycleManager.toWaiting())
+ .thenAnswer(invocation -> null);
+ workerWaitingStrategy.disconnect();
+ }
+
+ try (
+ MockedStatic<ServerLifeCycleManager> serverLifeCycleManagerMockedStatic =
+ Mockito.mockStatic(ServerLifeCycleManager.class)) {
+ Mockito.reset(workerConfig);
+ given(workerConfig.getRegistryDisconnectStrategy()).willThrow(new NullPointerException(""));
+ doNothing().when(stoppable).stop(anyString());
+ given(registryClient.getStoppable())
+ .willReturn(stoppable);
+ serverLifeCycleManagerMockedStatic
+ .when(() -> ServerLifeCycleManager.toWaiting())
+ .thenAnswer(invocation -> null);
+ workerWaitingStrategy.disconnect();
+ }
+ }
+}
(dolphinscheduler) 16/18: [Fix] Fix WorkflowInstance batch start failed will throw incorrect exception. (#15577)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 0b722999d35afd826acc774494cc3ac8626eca65
Author: 旺阳 <wa...@lqwang.net>
AuthorDate: Tue Feb 6 13:16:25 2024 +0800
[Fix] Fix WorkflowInstance batch start failed will throw incorrect exception. (#15577)
(cherry picked from commit 73a5a77db249cbfc69e564557bb067e6febda6d9)
---
.../dolphinscheduler/api/controller/ExecutorController.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
index 22d44a08bf..98070b4d65 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
@@ -18,10 +18,12 @@
package org.apache.dolphinscheduler.api.controller;
import static org.apache.dolphinscheduler.api.enums.Status.BATCH_EXECUTE_PROCESS_INSTANCE_ERROR;
+import static org.apache.dolphinscheduler.api.enums.Status.BATCH_START_PROCESS_INSTANCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.CHECK_PROCESS_DEFINITION_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.EXECUTE_PROCESS_INSTANCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.QUERY_EXECUTING_WORKFLOW_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.START_PROCESS_INSTANCE_ERROR;
+import static org.apache.dolphinscheduler.api.enums.Status.START_TASK_INSTANCE_ERROR;
import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.enums.Status;
@@ -226,7 +228,7 @@ public class ExecutorController extends BaseController {
})
@PostMapping(value = "batch-start-process-instance")
@ResponseStatus(HttpStatus.OK)
- @ApiException(START_PROCESS_INSTANCE_ERROR)
+ @ApiException(BATCH_START_PROCESS_INSTANCE_ERROR)
public Result batchStartProcessInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode,
@RequestParam(value = "processDefinitionCodes") String processDefinitionCodes,
@@ -292,7 +294,7 @@ public class ExecutorController extends BaseController {
}
if (!startFailedProcessDefinitionCodeList.isEmpty()) {
- putMsg(result, Status.BATCH_START_PROCESS_INSTANCE_ERROR,
+ putMsg(result, BATCH_START_PROCESS_INSTANCE_ERROR,
String.join(Constants.COMMA, startFailedProcessDefinitionCodeList));
}
@@ -437,7 +439,7 @@ public class ExecutorController extends BaseController {
})
@PostMapping(value = "/task-instance/{code}/start")
@ResponseStatus(HttpStatus.OK)
- @ApiException(START_PROCESS_INSTANCE_ERROR)
+ @ApiException(START_TASK_INSTANCE_ERROR)
public Result<Boolean> startStreamTaskInstance(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@Parameter(name = "projectCode", description = "PROJECT_CODE", required = true) @PathVariable long projectCode,
@Parameter(name = "code", description = "TASK_CODE", required = true) @PathVariable long code,
(dolphinscheduler) 04/18: [Doc][Docker] fix typo on start with docker (#15534)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 9e4e890f8476d5c10f073e9713b8b0bbafa4f115
Author: John Huang <pe...@gmail.com>
AuthorDate: Fri Feb 2 14:17:11 2024 +0800
[Doc][Docker] fix typo on start with docker (#15534)
Co-authored-by: xiangzihao <46...@qq.com>
Co-authored-by: Eric Gao <er...@gmail.com>
(cherry picked from commit 1b42d45fcdd22ccac7692fac4a64d096736f5e7c)
---
docs/docs/en/guide/start/docker.md | 8 ++++----
docs/docs/zh/guide/start/docker.md | 8 ++++----
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/docs/docs/en/guide/start/docker.md b/docs/docs/en/guide/start/docker.md
index 1e65946715..7163ff11f8 100644
--- a/docs/docs/en/guide/start/docker.md
+++ b/docs/docs/en/guide/start/docker.md
@@ -81,7 +81,7 @@ $ docker run -d --name dolphinscheduler-tools \
# Starting DolphinScheduler service
$ docker run -d --name dolphinscheduler-master \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
@@ -90,7 +90,7 @@ $ docker run -d --name dolphinscheduler-master \
-d apache/dolphinscheduler-master:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-worker \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
@@ -99,7 +99,7 @@ $ docker run -d --name dolphinscheduler-worker \
-d apache/dolphinscheduler-worker:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-api \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
@@ -108,7 +108,7 @@ $ docker run -d --name dolphinscheduler-api \
-d apache/dolphinscheduler-api:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-alert-server \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
diff --git a/docs/docs/zh/guide/start/docker.md b/docs/docs/zh/guide/start/docker.md
index 66e74656f4..8a21474a2f 100644
--- a/docs/docs/zh/guide/start/docker.md
+++ b/docs/docs/zh/guide/start/docker.md
@@ -74,7 +74,7 @@ $ docker run -d --name dolphinscheduler-tools \
# 启动 DolphinScheduler 对应的服务
$ docker run -d --name dolphinscheduler-master \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
@@ -83,7 +83,7 @@ $ docker run -d --name dolphinscheduler-master \
-d apache/dolphinscheduler-master:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-worker \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
@@ -92,7 +92,7 @@ $ docker run -d --name dolphinscheduler-worker \
-d apache/dolphinscheduler-worker:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-api \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
@@ -101,7 +101,7 @@ $ docker run -d --name dolphinscheduler-api \
-d apache/dolphinscheduler-api:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-alert-server \
-e DATABASE="postgresql" \
- -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
-e SPRING_DATASOURCE_USERNAME="<USER>" \
-e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
-e SPRING_JACKSON_TIME_ZONE="UTC" \
(dolphinscheduler) 03/18: Fix TaskGroupQueue will never be wakeup due to wakeup failed at one time (#15528)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit c8f120a60a0a503f43c89b2a499bf0c7ce8b905a
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Thu Feb 1 17:13:58 2024 +0800
Fix TaskGroupQueue will never be wakeup due to wakeup failed at one time (#15528)
(cherry picked from commit 89742332a8c19ee8d52a2dc786b460f4fc96ee62)
---
.../api/controller/TaskGroupController.java | 26 +-
.../api/service/TaskGroupService.java | 8 -
.../api/service/impl/ExecutorServiceImpl.java | 49 +--
.../service/impl/TaskGroupQueueServiceImpl.java | 18 +-
.../api/service/impl/TaskGroupServiceImpl.java | 23 +-
.../api/service/TaskGroupServiceTest.java | 9 +-
.../dolphinscheduler/dao/entity/TaskGroup.java | 33 +-
.../dao/mapper/TaskGroupMapper.java | 32 +-
.../dao/mapper/TaskGroupQueueMapper.java | 17 +
.../dao/repository/TaskGroupDao.java | 62 +++
.../dao/repository/TaskGroupQueueDao.java | 64 +++
.../dao/repository/impl/TaskGroupDaoImpl.java | 70 ++++
.../dao/repository/impl/TaskGroupQueueDaoImpl.java | 73 ++++
.../dao/mapper/TaskGroupMapper.xml | 58 ++-
.../dao/mapper/TaskGroupQueueMapper.xml | 61 ++-
.../src/main/resources/sql/dolphinscheduler_h2.sql | 1 +
.../main/resources/sql/dolphinscheduler_mysql.sql | 1 +
.../resources/sql/dolphinscheduler_postgresql.sql | 2 +
.../3.2.1_schema/mysql/dolphinscheduler_ddl.sql | 17 +-
.../postgresql/dolphinscheduler_ddl.sql | 4 +-
.../3.3.0_schema/mysql/dolphinscheduler_ddl.sql | 17 +-
.../postgresql/dolphinscheduler_ddl.sql | 4 +-
.../dao/mapper/TaskGroupMapperTest.java | 9 +-
.../dao/repository/impl/TaskGroupDaoImplTest.java | 118 ++++++
.../repository/impl/TaskGroupQueueDaoImplTest.java | 108 +++++
.../extract/master/ILogicTaskInstanceOperator.java | 9 -
.../extract/master/IWorkflowInstanceService.java | 5 +
.../transportor/TaskInstanceForceStartRequest.java | 42 --
.../TaskInstanceForceStartResponse.java | 41 --
.../transportor/TaskInstanceWakeupRequest.java | 19 +-
.../server/master/MasterServer.java | 5 +
.../master/event/StateEventHandlerManager.java | 10 +-
.../server/master/event/TaskStateEventHandler.java | 11 +-
.../event/TaskWaitTaskGroupStateHandler.java | 47 ---
.../LogicTaskInstanceOperationFunctionManager.java | 13 -
.../master/rpc/LogicTaskInstanceOperatorImpl.java | 14 -
.../TaskInstanceForceStartOperationFunction.java | 58 ---
.../rpc/TaskInstanceWakeupOperationFunction.java | 36 +-
.../master/rpc/WorkflowInstanceServiceImpl.java | 10 +
.../master/runner/WorkflowExecuteRunnable.java | 147 ++-----
.../runner/WorkflowExecuteRunnableFactory.java | 9 +-
.../runner/taskgroup/TaskGroupCoordinator.java | 451 +++++++++++++++++++++
.../master/runner/WorkflowExecuteRunnableTest.java | 11 +-
.../runner/taskgroup/TaskGroupCoordinatorTest.java | 182 +++++++++
.../registry/api/enums/RegistryNodeType.java | 1 +
.../service/process/ProcessService.java | 33 --
.../service/process/ProcessServiceImpl.java | 315 --------------
.../service/process/ProcessServiceTest.java | 16 -
48 files changed, 1452 insertions(+), 917 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java
index cb0d15909e..5d8735fbaa 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/TaskGroupController.java
@@ -298,7 +298,7 @@ public class TaskGroupController extends BaseController {
* @param pageSize page size
* @return queue list
*/
- @Operation(summary = "queryTasksByGroupId", description = "QUERY_ALL_TASKS_GROUP_NOTES")
+ @Operation(summary = "queryTaskGroupQueuesByGroupId", description = "QUERY_TASKS_GROUP_GROUP_QUEUES")
@Parameters({
@Parameter(name = "groupId", description = "GROUP_ID", required = false, schema = @Schema(implementation = int.class, example = "1", defaultValue = "-1")),
@Parameter(name = "taskInstanceName", description = "TASK_INSTANCE_NAME", required = false, schema = @Schema(implementation = String.class, example = "taskName")),
@@ -310,15 +310,21 @@ public class TaskGroupController extends BaseController {
@GetMapping(value = "/query-list-by-group-id")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_TASK_GROUP_QUEUE_LIST_ERROR)
- public Result queryTasksByGroupId(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam(value = "groupId", required = false, defaultValue = "-1") Integer groupId,
- @RequestParam(value = "taskInstanceName", required = false) String taskName,
- @RequestParam(value = "processInstanceName", required = false) String processName,
- @RequestParam(value = "status", required = false) Integer status,
- @RequestParam("pageNo") Integer pageNo,
- @RequestParam("pageSize") Integer pageSize) {
- Map<String, Object> result = taskGroupQueueService.queryTasksByGroupId(loginUser, taskName, processName, status,
- groupId, pageNo, pageSize);
+ public Result queryTaskGroupQueues(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+ @RequestParam(value = "groupId", required = false, defaultValue = "-1") Integer groupId,
+ @RequestParam(value = "taskInstanceName", required = false) String taskName,
+ @RequestParam(value = "processInstanceName", required = false) String processName,
+ @RequestParam(value = "status", required = false) Integer status,
+ @RequestParam("pageNo") Integer pageNo,
+ @RequestParam("pageSize") Integer pageSize) {
+ Map<String, Object> result = taskGroupQueueService.queryTasksByGroupId(
+ loginUser,
+ taskName,
+ processName,
+ status,
+ groupId,
+ pageNo,
+ pageSize);
return returnDataList(result);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java
index db909da158..885e23cf70 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/TaskGroupService.java
@@ -50,14 +50,6 @@ public interface TaskGroupService {
Map<String, Object> updateTaskGroup(User loginUser, int id, String name,
String description, int groupSize);
- /**
- * get task group status
- *
- * @param id task group id
- * @return the result code and msg
- */
- boolean isTheTaskGroupAvailable(int id);
-
/**
* query all task group by user id
*
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
index 25eca41158..7ab6102ff8 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
@@ -55,7 +55,6 @@ import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.enums.RunMode;
import org.apache.dolphinscheduler.common.enums.TaskDependType;
-import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.common.model.Server;
@@ -83,14 +82,12 @@ import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
-import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator;
import org.apache.dolphinscheduler.extract.master.IStreamingTaskOperator;
import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener;
import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService;
import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.service.command.CommandService;
@@ -552,16 +549,20 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
Map<String, Object> result = new HashMap<>();
TaskGroupQueue taskGroupQueue = taskGroupQueueMapper.selectById(queueId);
// check process instance exist
- ProcessInstance processInstance = processInstanceMapper.selectById(taskGroupQueue.getProcessId());
- if (processInstance == null) {
- log.error("Process instance does not exist, projectCode:{}, processInstanceId:{}.",
- taskGroupQueue.getProjectCode(), taskGroupQueue.getProcessId());
- putMsg(result, Status.PROCESS_INSTANCE_NOT_EXIST, taskGroupQueue.getProcessId());
- return result;
+ ProcessInstance processInstance = processInstanceDao.queryOptionalById(taskGroupQueue.getProcessId())
+ .orElseThrow(
+ () -> new ServiceException(Status.PROCESS_INSTANCE_NOT_EXIST, taskGroupQueue.getProcessId()));
+ checkMasterExists();
+
+ if (taskGroupQueue.getInQueue() == Flag.NO.getCode()) {
+ throw new ServiceException(Status.TASK_GROUP_QUEUE_ALREADY_START);
}
+ taskGroupQueue.setForceStart(Flag.YES.getCode());
+ taskGroupQueue.setUpdateTime(new Date());
+ taskGroupQueueMapper.updateById(taskGroupQueue);
- checkMasterExists();
- return forceStart(processInstance, taskGroupQueue);
+ result.put(Constants.STATUS, Status.SUCCESS);
+ return result;
}
public void checkStartNodeList(String startNodeList, Long processDefinitionCode, int version) {
@@ -664,32 +665,6 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
return result;
}
- /**
- * prepare to update process instance command type and status
- *
- * @param processInstance process instance
- * @return update result
- */
- private Map<String, Object> forceStart(ProcessInstance processInstance, TaskGroupQueue taskGroupQueue) {
- Map<String, Object> result = new HashMap<>();
- if (taskGroupQueue.getStatus() != TaskGroupQueueStatus.WAIT_QUEUE) {
- log.warn("Task group queue already starts, taskGroupQueueId:{}.", taskGroupQueue.getId());
- putMsg(result, Status.TASK_GROUP_QUEUE_ALREADY_START);
- return result;
- }
-
- taskGroupQueue.setForceStart(Flag.YES.getCode());
- taskGroupQueue.setUpdateTime(new Date());
- processService.updateTaskGroupQueue(taskGroupQueue);
- log.info("Sending force start command to master: {}.", processInstance.getHost());
- ILogicTaskInstanceOperator iLogicTaskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory
- .getProxyClient(processInstance.getHost(), ILogicTaskInstanceOperator.class);
- iLogicTaskInstanceOperator.forceStartTaskInstance(
- new TaskInstanceForceStartRequest(processInstance.getId(), taskGroupQueue.getTaskId()));
- putMsg(result, Status.SUCCESS);
- return result;
- }
-
/**
* check whether sub processes are offline before starting process definition
*
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java
index d33115479f..02c1bea4dc 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupQueueServiceImpl.java
@@ -66,8 +66,13 @@ public class TaskGroupQueueServiceImpl extends BaseServiceImpl implements TaskGr
* @return tasks list
*/
@Override
- public Map<String, Object> queryTasksByGroupId(User loginUser, String taskName, String processName, Integer status,
- int groupId, int pageNo, int pageSize) {
+ public Map<String, Object> queryTasksByGroupId(User loginUser,
+ String taskName,
+ String processName,
+ Integer status,
+ int groupId,
+ int pageNo,
+ int pageSize) {
Map<String, Object> result = new HashMap<>();
Page<TaskGroupQueue> page = new Page<>(pageNo, pageSize);
PageInfo<TaskGroupQueue> pageInfo = new PageInfo<>(pageNo, pageSize);
@@ -79,8 +84,13 @@ public class TaskGroupQueueServiceImpl extends BaseServiceImpl implements TaskGr
return result;
}
List<Project> projects = projectMapper.selectBatchIds(projectIds);
- IPage<TaskGroupQueue> taskGroupQueue = taskGroupQueueMapper.queryTaskGroupQueueByTaskGroupIdPaging(page,
- taskName, processName, status, groupId, projects);
+ IPage<TaskGroupQueue> taskGroupQueue = taskGroupQueueMapper.queryTaskGroupQueueByTaskGroupIdPaging(
+ page,
+ taskName,
+ processName,
+ status,
+ groupId,
+ projects);
pageInfo.setTotal((int) taskGroupQueue.getTotal());
pageInfo.setTotalList(taskGroupQueue.getRecords());
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java
index 3122d090e2..a1d9b4acf8 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskGroupServiceImpl.java
@@ -122,7 +122,7 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe
.description(description)
.groupSize(groupSize)
.userId(loginUser.getId())
- .status(Flag.YES.getCode())
+ .status(Flag.YES)
.createTime(now)
.updateTime(now)
.build();
@@ -180,7 +180,7 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe
putMsg(result, Status.TASK_GROUP_NAME_EXSIT);
return result;
}
- if (taskGroup.getStatus() != Flag.YES.getCode()) {
+ if (taskGroup.getStatus() != Flag.YES) {
log.warn("Task group has been closed, taskGroupId:{}.", id);
putMsg(result, Status.TASK_GROUP_STATUS_ERROR);
return result;
@@ -202,17 +202,6 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe
return result;
}
- /**
- * get task group status
- *
- * @param id task group id
- * @return is the task group available
- */
- @Override
- public boolean isTheTaskGroupAvailable(int id) {
- return taskGroupMapper.selectCountByIdStatus(id, Flag.YES.getCode()) == 1;
- }
-
/**
* query all task group by user id
*
@@ -331,12 +320,12 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe
return result;
}
TaskGroup taskGroup = taskGroupMapper.selectById(id);
- if (taskGroup.getStatus() == Flag.NO.getCode()) {
+ if (taskGroup.getStatus() == Flag.NO) {
log.info("Task group has been closed, taskGroupId:{}.", id);
putMsg(result, Status.TASK_GROUP_STATUS_CLOSED);
return result;
}
- taskGroup.setStatus(Flag.NO.getCode());
+ taskGroup.setStatus(Flag.NO);
int update = taskGroupMapper.updateById(taskGroup);
if (update > 0)
log.info("Task group close complete, taskGroupId:{}.", id);
@@ -364,12 +353,12 @@ public class TaskGroupServiceImpl extends BaseServiceImpl implements TaskGroupSe
return result;
}
TaskGroup taskGroup = taskGroupMapper.selectById(id);
- if (taskGroup.getStatus() == Flag.YES.getCode()) {
+ if (taskGroup.getStatus() == Flag.YES) {
log.info("Task group has been started, taskGroupId:{}.", id);
putMsg(result, Status.TASK_GROUP_STATUS_OPENED);
return result;
}
- taskGroup.setStatus(Flag.YES.getCode());
+ taskGroup.setStatus(Flag.YES);
taskGroup.setUpdateTime(new Date(System.currentTimeMillis()));
int update = taskGroupMapper.updateById(taskGroup);
if (update > 0)
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskGroupServiceTest.java
index ae50a54037..22de6c675a 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskGroupServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TaskGroupServiceTest.java
@@ -125,7 +125,7 @@ public class TaskGroupServiceTest {
.description(taskGroupDesc)
.groupSize(100)
.userId(1)
- .status(Flag.YES.getCode())
+ .status(Flag.YES)
.build();
return taskGroup;
@@ -204,7 +204,7 @@ public class TaskGroupServiceTest {
User loginUser = getLoginUser();
TaskGroup taskGroup = getTaskGroup();
- taskGroup.setStatus(Flag.YES.getCode());
+ taskGroup.setStatus(Flag.YES);
// Task group status error
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.TASK_GROUP,
@@ -218,7 +218,6 @@ public class TaskGroupServiceTest {
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
- taskGroup.setStatus(0);
}
@Test
@@ -236,12 +235,12 @@ public class TaskGroupServiceTest {
Map<String, Object> result = taskGroupService.closeTaskGroup(loginUser, 1);
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
- taskGroup.setStatus(0);
+ taskGroup.setStatus(Flag.NO);
Mockito.when(taskGroupMapper.selectById(1)).thenReturn(taskGroup);
result = taskGroupService.closeTaskGroup(loginUser, 1);
Assertions.assertEquals(Status.TASK_GROUP_STATUS_CLOSED, result.get(Constants.STATUS));
- taskGroup.setStatus(1);
+ taskGroup.setStatus(Flag.YES);
Mockito.when(taskGroupMapper.selectById(1)).thenReturn(taskGroup);
result = taskGroupService.startTaskGroup(loginUser, 1);
Assertions.assertEquals(Status.TASK_GROUP_STATUS_OPENED, result.get(Constants.STATUS));
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskGroup.java
index 3aa49f8e17..30e8a095af 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskGroup.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskGroup.java
@@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.dao.entity;
+import org.apache.dolphinscheduler.common.enums.Flag;
+
import java.io.Serializable;
import java.util.Date;
@@ -36,44 +38,17 @@ import com.baomidou.mybatisplus.annotation.TableName;
@TableName("t_ds_task_group")
public class TaskGroup implements Serializable {
- /**
- * key
- */
@TableId(value = "id", type = IdType.AUTO)
private Integer id;
- /**
- * task_group name
- */
private String name;
+ private long projectCode;
private String description;
- /**
- * 作业组大小
- */
private int groupSize;
- /**
- * 已使用作业组大小
- */
private int useSize;
- /**
- * creator id
- */
private int userId;
- /**
- * 0 not available, 1 available
- */
- private Integer status;
- /**
- * create time
- */
+ private Flag status;
private Date createTime;
- /**
- * update time
- */
private Date updateTime;
- /**
- * project code
- */
- private long projectCode;
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java
index 794bdbab3a..1c98eeff95 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.java
@@ -35,20 +35,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
*/
public interface TaskGroupMapper extends BaseMapper<TaskGroup> {
- int robTaskGroupResource(@Param("id") int id,
- @Param("currentUseSize") int currentUseSize,
- @Param("queueId") int queueId,
- @Param("queueStatus") int queueStatus);
-
- /**
- * update table of task group
- *
- * @param id primary key
- * @return affected rows
- */
- int releaseTaskGroupResource(@Param("id") int id, @Param("useSize") int useSize,
- @Param("queueId") int queueId, @Param("queueStatus") int queueStatus);
-
/**
* select task groups paging
*
@@ -57,7 +43,8 @@ public interface TaskGroupMapper extends BaseMapper<TaskGroup> {
* @param status status
* @return result page
*/
- IPage<TaskGroup> queryTaskGroupPaging(IPage<TaskGroup> page, @Param("name") String name,
+ IPage<TaskGroup> queryTaskGroupPaging(IPage<TaskGroup> page,
+ @Param("name") String name,
@Param("status") Integer status);
/**
@@ -69,13 +56,6 @@ public interface TaskGroupMapper extends BaseMapper<TaskGroup> {
*/
TaskGroup queryByName(@Param("userId") int userId, @Param("name") String name);
- /**
- * Select the groupSize > useSize Count
- */
- int selectAvailableCountById(@Param("groupId") int groupId);
-
- int selectCountByIdStatus(@Param("id") int id, @Param("status") int status);
-
IPage<TaskGroup> queryTaskGroupPagingByProjectCode(Page<TaskGroup> page, @Param("projectCode") Long projectCode);
/**
@@ -87,4 +67,12 @@ public interface TaskGroupMapper extends BaseMapper<TaskGroup> {
List<TaskGroup> listAuthorizedResource(@Param("userId") int userId);
List<TaskGroup> selectByProjectCode(@Param("projectCode") long projectCode);
+
+ List<TaskGroup> queryAvailableTaskGroups();
+
+ List<TaskGroup> queryUsedTaskGroups();
+
+ int acquireTaskGroupSlot(@Param("id") Integer id);
+
+ int releaseTaskGroupSlot(@Param("id") Integer id);
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java
index 437a22efb7..cada1c7092 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.java
@@ -104,5 +104,22 @@ public interface TaskGroupQueueMapper extends BaseMapper<TaskGroupQueue> {
void deleteByWorkflowInstanceId(@Param("workflowInstanceId") Integer workflowInstanceId);
+ void deleteByWorkflowInstanceIds(@Param("workflowInstanceIds") List<Integer> workflowInstanceIds);
+
void deleteByTaskGroupIds(@Param("taskGroupIds") List<Integer> taskGroupIds);
+
+ void updateTaskGroupPriorityByTaskInstanceId(@Param("taskInstanceId") Integer taskInstanceId,
+ @Param("priority") int taskGroupPriority);
+
+ List<TaskGroupQueue> queryAllInQueueTaskGroupQueueByGroupId(@Param("taskGroupId") Integer taskGroupId,
+ @Param("inQueue") int inQueue);
+
+ List<TaskGroupQueue> queryAllTaskGroupQueueByInQueue(@Param("inQueue") int inQueue);
+
+ List<TaskGroupQueue> queryByTaskInstanceId(@Param("taskInstanceId") Integer taskInstanceId);
+
+ List<TaskGroupQueue> queryUsingTaskGroupQueueByGroupId(@Param("taskGroupId") Integer taskGroupId,
+ @Param("status") int status,
+ @Param("inQueue") int inQueue,
+ @Param("forceStart") int forceStart);
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupDao.java
new file mode 100644
index 0000000000..ecdf68a2eb
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupDao.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository;
+
+import org.apache.dolphinscheduler.dao.entity.TaskGroup;
+
+import java.util.List;
+
+public interface TaskGroupDao extends IDao<TaskGroup> {
+
+ /**
+ * Query all TaskGroups
+ *
+ * @return all TaskGroups
+ */
+ List<TaskGroup> queryAllTaskGroups();
+
+ /**
+ * Query all TaskGroups which useSize > 0
+ *
+ * @return the TaskGroups which useSize > 0
+ */
+ List<TaskGroup> queryUsedTaskGroups();
+
+ /**
+ * Query all TaskGroups which useSize < groupSize
+ *
+ * @return the TaskGroups which useSize < groupSize
+ */
+ List<TaskGroup> queryAvailableTaskGroups();
+
+ /**
+ * Acquire a slot for the TaskGroup which useSize should < groupSize, set the useSize = useSize + 1.
+ *
+ * @param taskGroupId taskGroupId which shouldn't be null
+ * @return true if acquire successfully, false otherwise.
+ */
+ boolean acquireTaskGroupSlot(Integer taskGroupId);
+
+ /**
+ * Release a slot for the TaskGroup which useSize should > 0, set the useSize = useSize - 1.
+ *
+ * @param taskGroupId taskGroupId which shouldn't be null
+ * @return true if release successfully, false otherwise.
+ */
+ boolean releaseTaskGroupSlot(Integer taskGroupId);
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupQueueDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupQueueDao.java
new file mode 100644
index 0000000000..a788b29bb1
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TaskGroupQueueDao.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository;
+
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
+
+import java.util.List;
+
+public interface TaskGroupQueueDao extends IDao<TaskGroupQueue> {
+
+ /**
+ * Delete {@link TaskGroupQueue} by {@link ProcessInstance#getId()}
+ *
+ * @param workflowInstanceIds workflowInstanceIds
+ */
+ void deleteByWorkflowInstanceIds(List<Integer> workflowInstanceIds);
+
+ /**
+ * Query all {@link TaskGroupQueue} which in_queue is {@link org.apache.dolphinscheduler.common.enums.Flag#YES}
+ *
+ * @return TaskGroupQueue ordered by priority desc
+ */
+ List<TaskGroupQueue> queryAllInQueueTaskGroupQueue();
+
+ /**
+ * Query all {@link TaskGroupQueue} which in_queue is {@link org.apache.dolphinscheduler.common.enums.Flag#YES} and taskGroupId is taskGroupId
+ *
+ * @param taskGroupId taskGroupId
+ * @return TaskGroupQueue ordered by priority desc
+ */
+ List<TaskGroupQueue> queryAllInQueueTaskGroupQueueByGroupId(Integer taskGroupId);
+
+ /**
+ * Query all {@link TaskGroupQueue} which taskId is taskInstanceId
+ *
+ * @param taskInstanceId taskInstanceId
+ * @return TaskGroupQueue ordered by priority desc
+ */
+ List<TaskGroupQueue> queryByTaskInstanceId(Integer taskInstanceId);
+
+ /**
+ * Query all {@link TaskGroupQueue} which status is TaskGroupQueueStatus.ACQUIRE_SUCCESS and forceStart is {@link org.apache.dolphinscheduler.common.enums.Flag#NO}.
+ *
+ * @param taskGroupId taskGroupId
+ * @return TaskGroupQueue
+ */
+ List<TaskGroupQueue> queryAcquiredTaskGroupQueueByGroupId(Integer taskGroupId);
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImpl.java
new file mode 100644
index 0000000000..80d2c473d7
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImpl.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository.impl;
+
+import org.apache.dolphinscheduler.dao.entity.TaskGroup;
+import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper;
+import org.apache.dolphinscheduler.dao.repository.BaseDao;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupDao;
+
+import java.util.List;
+
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.stereotype.Repository;
+
+@Slf4j
+@Repository
+public class TaskGroupDaoImpl extends BaseDao<TaskGroup, TaskGroupMapper> implements TaskGroupDao {
+
+ public TaskGroupDaoImpl(@NonNull TaskGroupMapper taskGroupMapper) {
+ super(taskGroupMapper);
+ }
+
+ @Override
+ public List<TaskGroup> queryAllTaskGroups() {
+ return mybatisMapper.selectList(null);
+ }
+
+ @Override
+ public List<TaskGroup> queryUsedTaskGroups() {
+ return mybatisMapper.queryUsedTaskGroups();
+ }
+
+ @Override
+ public List<TaskGroup> queryAvailableTaskGroups() {
+ return mybatisMapper.queryAvailableTaskGroups();
+ }
+
+ @Override
+ public boolean acquireTaskGroupSlot(Integer taskGroupId) {
+ if (taskGroupId == null) {
+ throw new IllegalArgumentException("taskGroupId cannot be null");
+ }
+ return mybatisMapper.acquireTaskGroupSlot(taskGroupId) > 0;
+ }
+
+ @Override
+ public boolean releaseTaskGroupSlot(Integer taskGroupId) {
+ if (taskGroupId == null) {
+ throw new IllegalArgumentException("taskGroupId cannot be null");
+ }
+ return mybatisMapper.releaseTaskGroupSlot(taskGroupId) > 0;
+ }
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImpl.java
new file mode 100644
index 0000000000..a1808a9091
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImpl.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository.impl;
+
+import org.apache.dolphinscheduler.common.enums.Flag;
+import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
+import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
+import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
+import org.apache.dolphinscheduler.dao.repository.BaseDao;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupQueueDao;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.util.List;
+
+import lombok.NonNull;
+
+import org.springframework.stereotype.Repository;
+
+@Repository
+public class TaskGroupQueueDaoImpl extends BaseDao<TaskGroupQueue, TaskGroupQueueMapper> implements TaskGroupQueueDao {
+
+ public TaskGroupQueueDaoImpl(@NonNull TaskGroupQueueMapper taskGroupQueueMapper) {
+ super(taskGroupQueueMapper);
+ }
+
+ @Override
+ public void deleteByWorkflowInstanceIds(List<Integer> workflowInstanceIds) {
+ if (CollectionUtils.isEmpty(workflowInstanceIds)) {
+ return;
+ }
+ mybatisMapper.deleteByWorkflowInstanceIds(workflowInstanceIds);
+ }
+
+ @Override
+ public List<TaskGroupQueue> queryAllInQueueTaskGroupQueue() {
+ return mybatisMapper.queryAllTaskGroupQueueByInQueue(Flag.YES.getCode());
+ }
+
+ @Override
+ public List<TaskGroupQueue> queryAllInQueueTaskGroupQueueByGroupId(Integer taskGroupId) {
+ return mybatisMapper.queryAllInQueueTaskGroupQueueByGroupId(taskGroupId, Flag.YES.getCode());
+ }
+
+ @Override
+ public List<TaskGroupQueue> queryByTaskInstanceId(Integer taskInstanceId) {
+ return mybatisMapper.queryByTaskInstanceId(taskInstanceId);
+ }
+
+ @Override
+ public List<TaskGroupQueue> queryAcquiredTaskGroupQueueByGroupId(Integer taskGroupId) {
+ return mybatisMapper.queryUsingTaskGroupQueueByGroupId(
+ taskGroupId,
+ TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode(),
+ Flag.YES.getCode(),
+ Flag.NO.getCode());
+ }
+}
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.xml
index c3f1df0b0e..eb14e03dd4 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapper.xml
@@ -67,24 +67,6 @@
where project_code = #{projectCode}
</select>
- <!--modify data by id-->
- <update id="robTaskGroupResource">
- update t_ds_task_group
- set use_size = use_size + 1
- where id = #{id}
- and use_size < group_size
- and use_size = #{currentUseSize}
- and (select count(1) FROM t_ds_task_group_queue where id = #{queueId} and status = #{queueStatus}) = 1
- </update>
-
- <!--modify data by id-->
- <update id="releaseTaskGroupResource">
- update t_ds_task_group
- set use_size = use_size-1
- where id = #{id} and use_size > 0 and
- (select count(1) FROM t_ds_task_group_queue where id = #{queueId} and status = #{queueStatus} ) = 1
- </update>
-
<select id="queryByName" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroup">
select
<include refid="baseSql" />
@@ -93,30 +75,42 @@
user_id = #{userId} and name = #{name}
</select>
- <select id="selectAvailableCountById" resultType="java.lang.Integer">
+ <select id="listAuthorizedResource" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroup">
select
- count(1)
+ <include refid="baseSql" />
from t_ds_task_group
- where
- id = #{groupId} and use_size < group_size
+ where 1=1
+ <if test="userId != 0">
+ and user_id = #{userId}
+ </if>
</select>
- <select id="selectCountByIdStatus" resultType="java.lang.Integer">
+ <select id="queryAvailableTaskGroups" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroup">
select
- count(1)
+ <include refid="baseSql"/>
from t_ds_task_group
- where
- id = #{id} and status = #{status}
+ where use_size <![CDATA[ < ]]> group_size
</select>
- <select id="listAuthorizedResource" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroup">
+ <select id="queryUsedTaskGroups" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroup">
select
- <include refid="baseSql" />
+ <include refid="baseSql"/>
from t_ds_task_group
- where 1=1
- <if test="userId != 0">
- and user_id = #{userId}
- </if>
+ where use_size <![CDATA[ > ]]> 0
</select>
+ <update id="acquireTaskGroupSlot">
+ update t_ds_task_group
+ set use_size = use_size + 1
+ where id = #{id}
+ and use_size < group_size
+ </update>
+
+ <update id="releaseTaskGroupSlot">
+ update t_ds_task_group
+ set use_size = use_size - 1
+ where id = #{id}
+ and use_size > 0
+ </update>
+
</mapper>
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.xml
index a5bd80fbad..790ad7bfae 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskGroupQueueMapper.xml
@@ -46,7 +46,7 @@
and group_id = #{groupId}
</if>
</where>
- order by update_time desc
+ order by update_time desc, id desc
</select>
<select id="queryByStatus" resultMap="TaskGroupQueueMap" resultType="map">
@@ -126,9 +126,19 @@
<select id="queryTaskGroupQueueByTaskGroupIdPaging" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroupQueue">
select
- queue.id, queue.task_name, queue.group_id, queue.process_id, queue.priority, queue.status
- , queue.force_start, queue.create_time, queue.update_time,
- process.name as processInstanceName,p.name as projectName,p.code as projectCode
+ queue.id,
+ queue.task_name,
+ queue.group_id,
+ queue.process_id,
+ queue.priority,
+ queue.in_queue,
+ queue.status,
+ queue.force_start,
+ queue.create_time,
+ queue.update_time,
+ process.name as processInstanceName,
+ p.name as projectName,
+ p.code as projectCode
from t_ds_task_group_queue queue
left join t_ds_process_instance process on queue.process_id = process.id
left join t_ds_process_definition p_f on process.process_definition_code = p_f.code
@@ -171,6 +181,15 @@
where process_id = #{workflowInstanceId}
</delete>
+ <delete id="deleteByWorkflowInstanceIds">
+ delete
+ from t_ds_task_group_queue
+ where process_id in
+ <foreach collection="workflowInstanceIds" index="index" item="i" open="(" separator="," close=")">
+ #{i}
+ </foreach>
+ </delete>
+
<delete id="deleteByTaskGroupIds">
delete
from t_ds_task_group_queue
@@ -180,4 +199,38 @@
</foreach>
</delete>
+ <update id="updateTaskGroupPriorityByTaskInstanceId">
+ update t_ds_task_group_queue
+ set priority = #{priority}
+ where task_id = #{taskInstanceId}
+ </update>
+
+ <select id="queryAllInQueueTaskGroupQueueByGroupId" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroupQueue">
+ select
+ <include refid="baseSql" />
+ from t_ds_task_group_queue
+ where group_id = #{taskGroupId} and in_queue = #{inQueue} order by priority desc
+ </select>
+
+ <select id="queryAllTaskGroupQueueByInQueue" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroupQueue">
+ select
+ <include refid="baseSql" />
+ from t_ds_task_group_queue
+ where in_queue = #{inQueue} order by priority desc
+ </select>
+
+ <select id="queryByTaskInstanceId" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroupQueue">
+ select
+ <include refid="baseSql" />
+ from t_ds_task_group_queue
+ where task_id = #{taskInstanceId}
+ </select>
+
+ <select id="queryUsingTaskGroupQueueByGroupId" resultType="org.apache.dolphinscheduler.dao.entity.TaskGroupQueue">
+ select
+ <include refid="baseSql" />
+ from t_ds_task_group_queue
+ where group_id = #{taskGroupId} and status = #{status} and force_start = #{forceStart} and in_queue = #{inQueue}
+ </select>
+
</mapper>
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
index b1d9b60c36..7bc73096aa 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
@@ -1976,6 +1976,7 @@ CREATE TABLE t_ds_task_group_queue
in_queue int(4) DEFAULT '0' ,
create_time datetime DEFAULT NULL ,
update_time datetime DEFAULT NULL ,
+ KEY idx_t_ds_task_group_queue_in_queue (in_queue) ,
PRIMARY KEY (id)
);
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
index bd22e92ded..626f6f8b46 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
@@ -1964,6 +1964,7 @@ CREATE TABLE `t_ds_task_group_queue` (
`in_queue` tinyint(4) DEFAULT '0' COMMENT 'ready to get the queue by other task finish 0 NO ,1 YES',
`create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
`update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+ KEY `idx_t_ds_task_group_queue_in_queue` (`in_queue`),
PRIMARY KEY( `id` )
)ENGINE= INNODB AUTO_INCREMENT= 1 DEFAULT CHARSET= utf8 COLLATE = utf8_bin;
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
index 0232e01eeb..353b97da51 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
@@ -1946,6 +1946,8 @@ CREATE TABLE t_ds_task_group_queue (
PRIMARY KEY (id)
);
+create index idx_t_ds_task_group_queue_in_queue on t_ds_task_group_queue(in_queue);
+
--
-- Table structure for table t_ds_task_group
--
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/mysql/dolphinscheduler_ddl.sql
index 957f072a43..f4d7c510c1 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/mysql/dolphinscheduler_ddl.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/mysql/dolphinscheduler_ddl.sql
@@ -109,4 +109,19 @@ ALTER TABLE `t_ds_process_definition_log` MODIFY COLUMN `version` int NOT NULL D
ALTER TABLE `t_ds_process_instance` MODIFY COLUMN `process_definition_version` int NOT NULL DEFAULT 1 COMMENT "process definition version";
ALTER TABLE `t_ds_task_definition` MODIFY COLUMN `version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
ALTER TABLE `t_ds_task_definition_log` MODIFY COLUMN `version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
-ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `task_definition_version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
\ No newline at end of file
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `task_definition_version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
+
+-- create idx_t_ds_task_group_queue_in_queue on t_ds_task_group_queue
+DROP PROCEDURE IF EXISTS create_idx_t_ds_task_group_queue_in_queue;
+delimiter d//
+CREATE PROCEDURE create_idx_t_ds_task_group_queue_in_queue()
+BEGIN
+ DECLARE index_exists INT DEFAULT 0;
+ SELECT COUNT(*) INTO index_exists FROM information_schema.statistics WHERE table_schema = (SELECT DATABASE()) AND table_name = 't_ds_task_group_queue' AND index_name = 'idx_t_ds_task_group_queue_in_queue';
+ IF index_exists = 0 THEN CREATE INDEX idx_t_ds_task_group_queue_in_queue ON t_ds_task_group_queue(in_queue);
+END IF;
+END;
+d//
+delimiter ;
+CALL create_idx_t_ds_task_group_queue_in_queue;
+DROP PROCEDURE create_idx_t_ds_task_group_queue_in_queue;
\ No newline at end of file
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/postgresql/dolphinscheduler_ddl.sql
index ce48a3374e..157ad548d3 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/postgresql/dolphinscheduler_ddl.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.2.1_schema/postgresql/dolphinscheduler_ddl.sql
@@ -29,4 +29,6 @@ ALTER TABLE "t_ds_process_definition_log" ALTER COLUMN "version" SET DEFAULT 1;
ALTER TABLE "t_ds_task_definition" ALTER COLUMN "version" SET DEFAULT 1;
ALTER TABLE "t_ds_task_definition_log" ALTER COLUMN "version" SET DEFAULT 1;
ALTER TABLE "t_ds_process_instance" ALTER COLUMN "process_definition_version" SET NOT NULL, ALTER COLUMN "process_definition_version" SET DEFAULT 1;
-ALTER TABLE "t_ds_task_instance" ALTER COLUMN "task_definition_version" SET NOT NULL, ALTER COLUMN "task_definition_version" SET DEFAULT 1;
\ No newline at end of file
+ALTER TABLE "t_ds_task_instance" ALTER COLUMN "task_definition_version" SET NOT NULL, ALTER COLUMN "task_definition_version" SET DEFAULT 1;
+
+CREATE INDEX IF NOT EXISTS idx_t_ds_task_group_queue_in_queue ON t_ds_task_group_queue(in_queue);
\ No newline at end of file
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql
index 6c73e112da..657414b4a0 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql
@@ -80,4 +80,19 @@ ALTER TABLE `t_ds_process_definition_log` MODIFY COLUMN `version` int NOT NULL D
ALTER TABLE `t_ds_process_instance` MODIFY COLUMN `process_definition_version` int NOT NULL DEFAULT 1 COMMENT "process definition version";
ALTER TABLE `t_ds_task_definition` MODIFY COLUMN `version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
ALTER TABLE `t_ds_task_definition_log` MODIFY COLUMN `version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
-ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `task_definition_version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
\ No newline at end of file
+ALTER TABLE `t_ds_task_instance` MODIFY COLUMN `task_definition_version` int NOT NULL DEFAULT 1 COMMENT "task definition version";
+
+-- create idx_t_ds_task_group_queue_in_queue on t_ds_task_group_queue
+DROP PROCEDURE IF EXISTS create_idx_t_ds_task_group_queue_in_queue;
+delimiter d//
+CREATE PROCEDURE create_idx_t_ds_task_group_queue_in_queue()
+BEGIN
+ DECLARE index_exists INT DEFAULT 0;
+ SELECT COUNT(*) INTO index_exists FROM information_schema.statistics WHERE table_schema = (SELECT DATABASE()) AND table_name = 't_ds_task_group_queue' AND index_name = 'idx_t_ds_task_group_queue_in_queue';
+ IF index_exists = 0 THEN CREATE INDEX idx_t_ds_task_group_queue_in_queue ON t_ds_task_group_queue(in_queue);
+END IF;
+END;
+d//
+delimiter ;
+CALL create_idx_t_ds_task_group_queue_in_queue;
+DROP PROCEDURE create_idx_t_ds_task_group_queue_in_queue;
\ No newline at end of file
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql
index fd13ed3594..e984774010 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql
@@ -68,4 +68,6 @@ ALTER TABLE "t_ds_process_definition_log" ALTER COLUMN "version" SET DEFAULT 1;
ALTER TABLE "t_ds_task_definition" ALTER COLUMN "version" SET DEFAULT 1;
ALTER TABLE "t_ds_task_definition_log" ALTER COLUMN "version" SET DEFAULT 1;
ALTER TABLE "t_ds_process_instance" ALTER COLUMN "process_definition_version" SET NOT NULL, ALTER COLUMN "process_definition_version" SET DEFAULT 1;
-ALTER TABLE "t_ds_task_instance" ALTER COLUMN "task_definition_version" SET NOT NULL, ALTER COLUMN "task_definition_version" SET DEFAULT 1;
\ No newline at end of file
+ALTER TABLE "t_ds_task_instance" ALTER COLUMN "task_definition_version" SET NOT NULL, ALTER COLUMN "task_definition_version" SET DEFAULT 1;
+
+CREATE INDEX IF NOT EXISTS idx_t_ds_task_group_queue_in_queue ON t_ds_task_group_queue(in_queue);
\ No newline at end of file
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java
index 71c01bd95e..b457406baf 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/TaskGroupMapperTest.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.dao.mapper;
+import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.BaseDaoTest;
import org.apache.dolphinscheduler.dao.entity.TaskGroup;
@@ -24,8 +25,6 @@ import java.util.Date;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import com.baomidou.mybatisplus.core.metadata.IPage;
@@ -33,8 +32,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
public class TaskGroupMapperTest extends BaseDaoTest {
- private static final Logger logger = LoggerFactory.getLogger(TaskGroupMapperTest.class);
-
@Autowired
TaskGroupMapper taskGroupMapper;
@@ -46,7 +43,7 @@ public class TaskGroupMapperTest extends BaseDaoTest {
taskGroup.setName("task group");
taskGroup.setId(1);
taskGroup.setUserId(1);
- taskGroup.setStatus(1);
+ taskGroup.setStatus(Flag.YES);
taskGroup.setGroupSize(10);
taskGroup.setDescription("this is a task group");
Date date = new Date(System.currentTimeMillis());
@@ -88,7 +85,7 @@ public class TaskGroupMapperTest extends BaseDaoTest {
Page<TaskGroup> page = new Page(1, 3);
IPage<TaskGroup> taskGroupIPage = taskGroupMapper.queryTaskGroupPaging(
page,
- taskGroup.getName(), taskGroup.getStatus());
+ taskGroup.getName(), taskGroup.getStatus().getCode());
Assertions.assertEquals(taskGroupIPage.getTotal(), 1);
}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImplTest.java
new file mode 100644
index 0000000000..1f0cdf24e0
--- /dev/null
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupDaoImplTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository.impl;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.dolphinscheduler.common.enums.Flag;
+import org.apache.dolphinscheduler.dao.BaseDaoTest;
+import org.apache.dolphinscheduler.dao.entity.TaskGroup;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupDao;
+
+import java.util.Date;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+class TaskGroupDaoImplTest extends BaseDaoTest {
+
+ @Autowired
+ private TaskGroupDao taskGroupDao;
+
+ @Test
+ void queryAllTaskGroups() {
+ TaskGroup taskGroup = createTaskGroup("test", 0, 1);
+ taskGroupDao.insert(taskGroup);
+ List<TaskGroup> taskGroups = taskGroupDao.queryAllTaskGroups();
+ assertEquals(1, taskGroups.size());
+ }
+
+ @Test
+ void queryUsedTaskGroups() {
+ // Insert a unused task group
+ TaskGroup taskGroup = createTaskGroup("testUnused", 0, 1);
+ taskGroupDao.insert(taskGroup);
+ assertEquals(0, taskGroupDao.queryUsedTaskGroups().size());
+
+ // Insert a used task group
+ taskGroup = createTaskGroup("testUsed", 1, 1);
+ taskGroupDao.insert(taskGroup);
+ assertEquals(1, taskGroupDao.queryUsedTaskGroups().size());
+ }
+
+ @Test
+ void queryAvailableTaskGroups() {
+ // Insert a full task group
+ TaskGroup taskGroup = createTaskGroup("testFull", 1, 1);
+ taskGroupDao.insert(taskGroup);
+ assertEquals(0, taskGroupDao.queryAvailableTaskGroups().size());
+
+ // Insert a used task group
+ taskGroup = createTaskGroup("testNotFull", 0, 1);
+ taskGroupDao.insert(taskGroup);
+ assertEquals(1, taskGroupDao.queryAvailableTaskGroups().size());
+ }
+
+ @Test
+ void acquireTaskGroupSlot() {
+ // Insert a full task group will acquire failed
+ TaskGroup taskGroup = createTaskGroup("testFull", 1, 1);
+ taskGroupDao.insert(taskGroup);
+ assertFalse(taskGroupDao.acquireTaskGroupSlot(taskGroup.getId()));
+
+ taskGroup.setUseSize(0);
+ taskGroupDao.updateById(taskGroup);
+ assertTrue(taskGroupDao.acquireTaskGroupSlot(taskGroup.getId()));
+
+ taskGroup = taskGroupDao.queryById(taskGroup.getId());
+ assertEquals(1, taskGroup.getUseSize());
+ }
+
+ @Test
+ void releaseTaskGroupSlot() {
+ // Insert an empty task group will release failed
+ TaskGroup taskGroup = createTaskGroup("testEmpty", 0, 1);
+ taskGroupDao.insert(taskGroup);
+ assertFalse(taskGroupDao.releaseTaskGroupSlot(taskGroup.getId()));
+
+ taskGroup.setUseSize(1);
+ taskGroupDao.updateById(taskGroup);
+ assertTrue(taskGroupDao.releaseTaskGroupSlot(taskGroup.getId()));
+
+ taskGroup = taskGroupDao.queryById(taskGroup.getId());
+ assertEquals(0, taskGroup.getUseSize());
+ }
+
+ private TaskGroup createTaskGroup(String name, int useSize, int groupSize) {
+ return TaskGroup.builder()
+ .name(name)
+ .description("test")
+ .groupSize(groupSize)
+ .useSize(useSize)
+ .userId(1)
+ .status(Flag.YES)
+ .createTime(new Date())
+ .updateTime(new Date())
+ .projectCode(1)
+ .build();
+ }
+
+}
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java
new file mode 100644
index 0000000000..17c1537184
--- /dev/null
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/TaskGroupQueueDaoImplTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.repository.impl;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.apache.dolphinscheduler.common.enums.Flag;
+import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
+import org.apache.dolphinscheduler.dao.BaseDaoTest;
+import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupQueueDao;
+
+import java.util.Date;
+
+import org.assertj.core.util.Lists;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+class TaskGroupQueueDaoImplTest extends BaseDaoTest {
+
+ @Autowired
+ private TaskGroupQueueDao taskGroupQueueDao;
+
+ @Test
+ void deleteByWorkflowInstanceIds() {
+ TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS);
+ taskGroupQueueDao.insert(taskGroupQueue);
+ assertNotNull(taskGroupQueueDao.queryById(taskGroupQueue.getId()));
+
+ taskGroupQueueDao.deleteByWorkflowInstanceIds(Lists.newArrayList(1));
+ assertNull(taskGroupQueueDao.queryById(taskGroupQueue.getId()));
+ }
+
+ @Test
+ void queryAllInQueueTaskGroupQueue() {
+ TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS);
+ taskGroupQueueDao.insert(taskGroupQueue);
+ assertEquals(1, taskGroupQueueDao.queryAllInQueueTaskGroupQueue().size());
+ }
+
+ @Test
+ void queryAllInQueueTaskGroupQueueByGroupId() {
+ TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS);
+ taskGroupQueueDao.insert(taskGroupQueue);
+ assertEquals(1, taskGroupQueueDao.queryAllInQueueTaskGroupQueueByGroupId(1).size());
+ }
+
+ @Test
+ void updateById() {
+ TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.WAIT_QUEUE);
+ taskGroupQueueDao.insert(taskGroupQueue);
+
+ taskGroupQueue.setStatus(TaskGroupQueueStatus.ACQUIRE_SUCCESS);
+ taskGroupQueueDao.updateById(taskGroupQueue);
+ assertEquals(TaskGroupQueueStatus.ACQUIRE_SUCCESS,
+ taskGroupQueueDao.queryById(taskGroupQueue.getId()).getStatus());
+ }
+
+ @Test
+ void queryByTaskInstanceId() {
+ TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS);
+ taskGroupQueueDao.insert(taskGroupQueue);
+ assertEquals(1, taskGroupQueueDao.queryByTaskInstanceId(1).size());
+ }
+
+ @Test
+ void queryUsingTaskGroupQueueByGroupId() {
+ TaskGroupQueue taskGroupQueue = createTaskGroupQueue(Flag.NO, TaskGroupQueueStatus.ACQUIRE_SUCCESS);
+ taskGroupQueueDao.insert(taskGroupQueue);
+ assertEquals(1, taskGroupQueueDao.queryAcquiredTaskGroupQueueByGroupId(1).size());
+
+ taskGroupQueue = createTaskGroupQueue(Flag.YES, TaskGroupQueueStatus.WAIT_QUEUE);
+ taskGroupQueueDao.insert(taskGroupQueue);
+ assertEquals(1, taskGroupQueueDao.queryAcquiredTaskGroupQueueByGroupId(1).size());
+ }
+
+ private TaskGroupQueue createTaskGroupQueue(Flag forceStart, TaskGroupQueueStatus taskGroupQueueStatus) {
+ return TaskGroupQueue.builder()
+ .taskId(1)
+ .taskName("test")
+ .groupId(1)
+ .processId(1)
+ .priority(0)
+ .forceStart(forceStart.getCode())
+ .inQueue(Flag.YES.getCode())
+ .status(taskGroupQueueStatus)
+ .createTime(new Date())
+ .updateTime(new Date())
+ .build();
+ }
+}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java
index a4fb547028..f85200b7a2 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java
@@ -25,9 +25,6 @@ import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillReque
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
@RpcService
public interface ILogicTaskInstanceOperator {
@@ -41,10 +38,4 @@ public interface ILogicTaskInstanceOperator {
@RpcMethod
LogicTaskPauseResponse pauseLogicTask(LogicTaskPauseRequest taskPauseRequest);
- @RpcMethod
- TaskInstanceForceStartResponse forceStartTaskInstance(TaskInstanceForceStartRequest taskForceStartRequest);
-
- @RpcMethod
- void wakeupTaskInstance(TaskInstanceWakeupRequest taskWakeupRequest);
-
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java
index 217e81322e..a535f8c6ca 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java
@@ -20,6 +20,8 @@ package org.apache.dolphinscheduler.extract.master;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
+import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
+import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupResponse;
@RpcService
public interface IWorkflowInstanceService {
@@ -30,4 +32,7 @@ public interface IWorkflowInstanceService {
@RpcMethod
WorkflowExecuteDto getWorkflowExecutingData(Integer workflowInstanceId);
+ @RpcMethod
+ TaskInstanceWakeupResponse wakeupTaskInstance(TaskInstanceWakeupRequest taskWakeupRequest);
+
}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java
deleted file mode 100644
index 0a4711fa1e..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartRequest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.master.transportor;
-
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-public class TaskInstanceForceStartRequest {
-
- private String key;
-
- private int processInstanceId;
-
- private int taskInstanceId;
-
- public TaskInstanceForceStartRequest(
- int processInstanceId,
- int taskInstanceId) {
- this.key = String.format("%d-%d", processInstanceId, taskInstanceId);
-
- this.processInstanceId = processInstanceId;
- this.taskInstanceId = taskInstanceId;
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java
deleted file mode 100644
index db19c023dc..0000000000
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceForceStartResponse.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.extract.master.transportor;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@NoArgsConstructor
-@AllArgsConstructor
-public class TaskInstanceForceStartResponse {
-
- private boolean success;
-
- private String message;
-
- public static TaskInstanceForceStartResponse success() {
- return new TaskInstanceForceStartResponse(true, "dispatch success");
- }
-
- public static TaskInstanceForceStartResponse failed(String message) {
- return new TaskInstanceForceStartResponse(false, message);
- }
-
-}
diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java
index 5bfbc03f1c..ddca440436 100644
--- a/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java
+++ b/dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/TaskInstanceWakeupRequest.java
@@ -17,26 +17,21 @@
package org.apache.dolphinscheduler.extract.master.transportor;
+import java.io.Serializable;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
+@Builder
@NoArgsConstructor
-public class TaskInstanceWakeupRequest {
-
- private String key;
+@AllArgsConstructor
+public class TaskInstanceWakeupRequest implements Serializable {
private int processInstanceId;
private int taskInstanceId;
- public TaskInstanceWakeupRequest(
- int processInstanceId,
- int taskInstanceId) {
- this.key = String.format("%d-%d", processInstanceId, taskInstanceId);
-
- this.processInstanceId = processInstanceId;
- this.taskInstanceId = taskInstanceId;
- }
-
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 7d6600cc43..d37ca9d016 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.server.master.rpc.MasterRpcServer;
import org.apache.dolphinscheduler.server.master.runner.EventExecuteService;
import org.apache.dolphinscheduler.server.master.runner.FailoverExecuteThread;
import org.apache.dolphinscheduler.server.master.runner.MasterSchedulerBootstrap;
+import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import javax.annotation.PostConstruct;
@@ -84,6 +85,9 @@ public class MasterServer implements IStoppable {
@Autowired
private MasterSlotManager masterSlotManager;
+ @Autowired
+ private TaskGroupCoordinator taskGroupCoordinator;
+
public static void main(String[] args) {
MasterServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount);
@@ -115,6 +119,7 @@ public class MasterServer implements IStoppable {
this.failoverExecuteThread.start();
this.schedulerApi.start();
+ this.taskGroupCoordinator.start();
MasterServerMetrics.registerMasterCpuUsageGauge(() -> {
SystemMetrics systemMetrics = metricsProvider.getSystemMetrics();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java
index b91da8cc51..7cd89b91fa 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandlerManager.java
@@ -24,14 +24,20 @@ import java.util.Map;
import java.util.Optional;
import java.util.ServiceLoader;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class StateEventHandlerManager {
private static final Map<StateEventType, StateEventHandler> stateEventHandlerMap = new HashMap<>();
static {
ServiceLoader.load(StateEventHandler.class)
- .forEach(stateEventHandler -> stateEventHandlerMap.put(stateEventHandler.getEventType(),
- stateEventHandler));
+ .forEach(stateEventHandler -> {
+ log.info("Initialize StateEventHandler: {} for eventType: {}",
+ stateEventHandler.getClass().getName(), stateEventHandler.getEventType());
+ stateEventHandlerMap.put(stateEventHandler.getEventType(), stateEventHandler);
+ });
}
public static Optional<StateEventHandler> getStateEventHandler(StateEventType stateEventType) {
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java
index f790131a70..2dde1315de 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskStateEventHandler.java
@@ -62,7 +62,8 @@ public class TaskStateEventHandler implements StateEventHandler {
if (task.getState().isFinished()
&& (taskStateEvent.getStatus() != null && taskStateEvent.getStatus().isRunning())) {
String errorMessage = String.format(
- "The current task instance state is %s, but the task state event status is %s, so the task state event will be ignored",
+ "The current TaskInstance: %s state is %s, but the task state event status is %s, so the task state event will be ignored",
+ task.getName(),
task.getState().name(),
taskStateEvent.getStatus().name());
log.warn(errorMessage);
@@ -75,14 +76,6 @@ public class TaskStateEventHandler implements StateEventHandler {
return true;
}
workflowExecuteRunnable.taskFinished(task);
- if (task.getTaskGroupId() > 0) {
- log.info("The task instance need to release task Group: {}", task.getTaskGroupId());
- try {
- workflowExecuteRunnable.releaseTaskGroup(task);
- } catch (Exception e) {
- throw new StateEventHandleException("Release task group failed", e);
- }
- }
return true;
}
return true;
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
deleted file mode 100644
index 4859e896b9..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.event;
-
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
-
-import lombok.extern.slf4j.Slf4j;
-
-import com.google.auto.service.AutoService;
-
-@AutoService(StateEventHandler.class)
-@Slf4j
-public class TaskWaitTaskGroupStateHandler implements StateEventHandler {
-
- @Override
- public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
- StateEvent stateEvent) {
- log.info("Handle task instance wait task group event, taskInstanceId: {}", stateEvent.getTaskInstanceId());
- if (workflowExecuteRunnable.checkForceStartAndWakeUp(stateEvent)) {
- log.info("Success wake up task instance, taskInstanceId: {}", stateEvent.getTaskInstanceId());
- } else {
- log.info("Failed to wake up task instance, taskInstanceId: {}", stateEvent.getTaskInstanceId());
- }
- return true;
- }
-
- @Override
- public StateEventType getEventType() {
- return StateEventType.WAKE_UP_TASK_GROUP;
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java
index 3a07d68364..de4095831e 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperationFunctionManager.java
@@ -32,12 +32,6 @@ public class LogicTaskInstanceOperationFunctionManager {
@Autowired
private LogicITaskInstancePauseOperationFunction logicITaskInstancePauseOperationFunction;
- @Autowired
- private TaskInstanceForceStartOperationFunction taskInstanceForceStartOperationFunction;
-
- @Autowired
- private TaskInstanceWakeupOperationFunction taskInstanceWakeupOperationFunction;
-
public LogicITaskInstanceDispatchOperationFunction getLogicTaskInstanceDispatchOperationFunction() {
return logicITaskInstanceDispatchOperationFunction;
}
@@ -50,11 +44,4 @@ public class LogicTaskInstanceOperationFunctionManager {
return logicITaskInstancePauseOperationFunction;
}
- public TaskInstanceForceStartOperationFunction getTaskInstanceForceStartOperationFunction() {
- return taskInstanceForceStartOperationFunction;
- }
-
- public TaskInstanceWakeupOperationFunction getTaskInstanceWakeupOperationFunction() {
- return taskInstanceWakeupOperationFunction;
- }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java
index c99968d1e9..39446bd89d 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/LogicTaskInstanceOperatorImpl.java
@@ -24,9 +24,6 @@ import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillReque
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartResponse;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
import lombok.extern.slf4j.Slf4j;
@@ -58,15 +55,4 @@ public class LogicTaskInstanceOperatorImpl implements ILogicTaskInstanceOperator
.operate(taskPauseRequest);
}
- @Override
- public TaskInstanceForceStartResponse forceStartTaskInstance(TaskInstanceForceStartRequest taskForceStartRequest) {
- return logicTaskInstanceOperationFunctionManager.getTaskInstanceForceStartOperationFunction()
- .operate(taskForceStartRequest);
- }
-
- @Override
- public void wakeupTaskInstance(TaskInstanceWakeupRequest taskWakeupRequest) {
- logicTaskInstanceOperationFunctionManager.getTaskInstanceWakeupOperationFunction().operate(taskWakeupRequest);
- }
-
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceForceStartOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceForceStartOperationFunction.java
deleted file mode 100644
index a15766e4c4..0000000000
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceForceStartOperationFunction.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.master.rpc;
-
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartResponse;
-import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
-import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
-import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-@Slf4j
-@Component
-public class TaskInstanceForceStartOperationFunction
- implements
- ITaskInstanceOperationFunction<TaskInstanceForceStartRequest, TaskInstanceForceStartResponse> {
-
- @Autowired
- private StateEventResponseService stateEventResponseService;
-
- @Override
- public TaskInstanceForceStartResponse operate(TaskInstanceForceStartRequest taskInstanceForceStartRequest) {
- TaskStateEvent stateEvent = TaskStateEvent.builder()
- .processInstanceId(taskInstanceForceStartRequest.getProcessInstanceId())
- .taskInstanceId(taskInstanceForceStartRequest.getTaskInstanceId())
- .key(taskInstanceForceStartRequest.getKey())
- .type(StateEventType.WAKE_UP_TASK_GROUP)
- .build();
- try {
- LogUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
- log.info("Received forceStartTaskInstance, event: {}", stateEvent);
- stateEventResponseService.addEvent2WorkflowExecute(stateEvent);
- return TaskInstanceForceStartResponse.success();
- } finally {
- LogUtils.removeWorkflowAndTaskInstanceIdMDC();
- }
- }
-}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceWakeupOperationFunction.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceWakeupOperationFunction.java
index 9a5b716e90..4a83874fb9 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceWakeupOperationFunction.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/TaskInstanceWakeupOperationFunction.java
@@ -17,12 +17,12 @@
package org.apache.dolphinscheduler.server.master.rpc;
-import org.apache.dolphinscheduler.common.enums.StateEventType;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupResponse;
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
-import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
-import org.apache.dolphinscheduler.server.master.processor.queue.StateEventResponseService;
+import org.apache.dolphinscheduler.server.master.cache.ProcessInstanceExecCacheManager;
+import org.apache.dolphinscheduler.server.master.runner.DefaultTaskExecuteRunnable;
+import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
import lombok.extern.slf4j.Slf4j;
@@ -36,20 +36,30 @@ public class TaskInstanceWakeupOperationFunction
ITaskInstanceOperationFunction<TaskInstanceWakeupRequest, TaskInstanceWakeupResponse> {
@Autowired
- private StateEventResponseService stateEventResponseService;
+ private ProcessInstanceExecCacheManager processInstanceExecCacheManager;
@Override
public TaskInstanceWakeupResponse operate(TaskInstanceWakeupRequest taskInstanceWakeupRequest) {
- TaskStateEvent stateEvent = TaskStateEvent.builder()
- .processInstanceId(taskInstanceWakeupRequest.getProcessInstanceId())
- .taskInstanceId(taskInstanceWakeupRequest.getTaskInstanceId())
- .key(taskInstanceWakeupRequest.getKey())
- .type(StateEventType.WAKE_UP_TASK_GROUP)
- .build();
try {
- LogUtils.setWorkflowAndTaskInstanceIDMDC(stateEvent.getProcessInstanceId(), stateEvent.getTaskInstanceId());
- log.info("Received wakeupTaskInstance request, event: {}", stateEvent);
- stateEventResponseService.addEvent2WorkflowExecute(stateEvent);
+ log.info("Received TaskInstanceWakeupRequest request{}", taskInstanceWakeupRequest);
+
+ int workflowInstanceId = taskInstanceWakeupRequest.getProcessInstanceId();
+ int taskInstanceId = taskInstanceWakeupRequest.getTaskInstanceId();
+ LogUtils.setWorkflowAndTaskInstanceIDMDC(workflowInstanceId, taskInstanceId);
+ WorkflowExecuteRunnable workflowExecuteRunnable =
+ processInstanceExecCacheManager.getByProcessInstanceId(workflowInstanceId);
+ if (workflowExecuteRunnable == null) {
+ log.warn("cannot find WorkflowExecuteRunnable: {}, no need to Wakeup task", workflowInstanceId);
+ return TaskInstanceWakeupResponse.failed("cannot find WorkflowExecuteRunnable: " + workflowInstanceId);
+ }
+ DefaultTaskExecuteRunnable defaultTaskExecuteRunnable =
+ workflowExecuteRunnable.getTaskExecuteRunnableById(taskInstanceId).orElse(null);
+ if (defaultTaskExecuteRunnable == null) {
+ log.warn("Cannot find DefaultTaskExecuteRunnable: {}, cannot Wakeup task", taskInstanceId);
+ return TaskInstanceWakeupResponse.failed("Cannot find DefaultTaskExecuteRunnable: " + taskInstanceId);
+ }
+ defaultTaskExecuteRunnable.dispatch();
+ log.info("Success Wakeup TaskInstance: {}", taskInstanceId);
return TaskInstanceWakeupResponse.success();
} finally {
LogUtils.removeWorkflowAndTaskInstanceIdMDC();
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java
index c3420cb1da..d10c8b81c4 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/rpc/WorkflowInstanceServiceImpl.java
@@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.server.master.rpc;
import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService;
import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
+import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
+import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupResponse;
import org.apache.dolphinscheduler.server.master.metrics.ProcessInstanceMetrics;
import org.apache.dolphinscheduler.server.master.service.ExecutingService;
@@ -36,6 +38,9 @@ public class WorkflowInstanceServiceImpl implements IWorkflowInstanceService {
@Autowired
private ExecutingService executingService;
+ @Autowired
+ private TaskInstanceWakeupOperationFunction taskInstanceWakeupOperationFunction;
+
@Override
public void clearWorkflowMetrics(Long workflowDefinitionCode) {
log.info("Receive clearWorkflowMetrics request: {}", workflowDefinitionCode);
@@ -49,4 +54,9 @@ public class WorkflowInstanceServiceImpl implements IWorkflowInstanceService {
executingService.queryWorkflowExecutingData(workflowInstanceId);
return workflowExecuteDtoOptional.orElse(null);
}
+
+ @Override
+ public TaskInstanceWakeupResponse wakeupTaskInstance(TaskInstanceWakeupRequest taskWakeupRequest) {
+ return taskInstanceWakeupOperationFunction.operate(taskWakeupRequest);
+ }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 3759f2209c..2db49c6f09 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -38,7 +38,6 @@ import org.apache.dolphinscheduler.common.enums.FailureStrategy;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.Priority;
import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.log.remote.RemoteLogUtils;
@@ -52,14 +51,11 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.Schedule;
-import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
-import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator;
-import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator;
import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse;
@@ -81,6 +77,7 @@ import org.apache.dolphinscheduler.server.master.event.WorkflowStateEvent;
import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph;
import org.apache.dolphinscheduler.server.master.metrics.TaskMetrics;
import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory;
+import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator;
import org.apache.dolphinscheduler.server.master.utils.TaskUtils;
import org.apache.dolphinscheduler.server.master.utils.WorkflowInstanceUtils;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
@@ -114,7 +111,6 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import lombok.NonNull;
@@ -227,6 +223,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
private final ListenerEventAlertManager listenerEventAlertManager;
+ private final TaskGroupCoordinator taskGroupCoordinator;
+
public WorkflowExecuteRunnable(
@NonNull IWorkflowExecuteContext workflowExecuteContext,
@NonNull CommandService commandService,
@@ -238,7 +236,8 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
@NonNull CuringParamsService curingParamsService,
@NonNull TaskInstanceDao taskInstanceDao,
@NonNull DefaultTaskExecuteRunnableFactory defaultTaskExecuteRunnableFactory,
- @NonNull ListenerEventAlertManager listenerEventAlertManager) {
+ @NonNull ListenerEventAlertManager listenerEventAlertManager,
+ @NonNull TaskGroupCoordinator taskGroupCoordinator) {
this.processService = processService;
this.commandService = commandService;
this.processInstanceDao = processInstanceDao;
@@ -250,6 +249,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
this.taskInstanceDao = taskInstanceDao;
this.defaultTaskExecuteRunnableFactory = defaultTaskExecuteRunnableFactory;
this.listenerEventAlertManager = listenerEventAlertManager;
+ this.taskGroupCoordinator = taskGroupCoordinator;
TaskMetrics.registerTaskPrepared(standByTaskInstancePriorityQueue::size);
}
@@ -339,43 +339,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
return this.stateEvents.size();
}
- public boolean checkForceStartAndWakeUp(StateEvent stateEvent) {
- TaskGroupQueue taskGroupQueue = processService.loadTaskGroupQueue(stateEvent.getTaskInstanceId());
- if (taskGroupQueue.getForceStart() == Flag.YES.getCode()) {
- log.info("Begin to force start taskGroupQueue: {}", taskGroupQueue.getId());
- TaskInstance taskInstance = taskInstanceDao.queryById(stateEvent.getTaskInstanceId());
-
- DefaultTaskExecuteRunnable defaultTaskExecuteRunnable =
- taskExecuteRunnableMap.get(taskInstance.getTaskCode());
- if (defaultTaskExecuteRunnable != null) {
- defaultTaskExecuteRunnable.dispatch();
- this.processService.updateTaskGroupQueueStatus(taskGroupQueue.getTaskId(),
- TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode());
- log.info("Success force start task: {}, taskGroup: {}", taskGroupQueue.getTaskName(),
- taskGroupQueue.getGroupId());
- } else {
- log.warn("Cannot find the TaskExecuteRunnable: {}", taskGroupQueue.getTaskName());
- }
- return true;
- }
- if (taskGroupQueue.getInQueue() == Flag.YES.getCode()) {
- log.info("Begin to wake up taskGroupQueue: {}", taskGroupQueue.getId());
- boolean acquireTaskGroup = processService.robTaskGroupResource(taskGroupQueue);
- if (acquireTaskGroup) {
- TaskInstance taskInstance = taskInstanceDao.queryById(stateEvent.getTaskInstanceId());
- taskExecuteRunnableMap.get(taskInstance.getTaskCode()).dispatch();
- log.info("Success wake up taskGroupQueue: {}", taskGroupQueue.getId());
- return true;
- }
- log.warn("Failed to wake up taskGroupQueue, taskGroupQueueId: {}", taskGroupQueue.getId());
- return false;
- } else {
- log.info(
- "Failed to wake up the taskGroupQueue: {}, since the taskGroupQueue is not in queue, will no need to wake up.",
- taskGroupQueue);
- return true;
- }
- }
public void processStart() {
ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
ProjectUser projectUser = processService.queryProjectWithUserByProcessInstanceId(workflowInstance.getId());
@@ -408,6 +371,11 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
taskExecuteRunnableMap.remove(taskInstance.getTaskCode());
stateWheelExecuteThread.removeTask4TimeoutCheck(workflowInstance, taskInstance);
stateWheelExecuteThread.removeTask4RetryCheck(workflowInstance, taskInstance);
+ if (taskInstance.getTaskGroupId() > 0) {
+ releaseTaskGroupIfNeeded(taskInstance);
+ log.info("Release task Group slot: {} for taskInstance: {} ", taskInstance.getTaskGroupId(),
+ taskInstance.getId());
+ }
if (taskInstance.getState().isSuccess()) {
completeTaskSet.add(taskInstance.getTaskCode());
@@ -462,51 +430,16 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
}
}
- /**
- * release task group
- *
- */
- public void releaseTaskGroup(TaskInstance taskInstance) {
- ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
+ private void releaseTaskGroupIfNeeded(TaskInstance taskInstance) {
// todo: use Integer
if (taskInstance.getTaskGroupId() <= 0) {
log.info("The current TaskInstance: {} doesn't use taskGroup, no need to release taskGroup",
taskInstance.getName());
return;
}
- TaskInstance nextTaskInstance = processService.releaseTaskGroup(taskInstance);
- if (nextTaskInstance == null) {
- log.info(
- "The current TaskInstance: {} is the last taskInstance in the taskGroup, no need to wakeup next taskInstance",
- taskInstance.getName());
- return;
- }
- if (nextTaskInstance.getProcessInstanceId() == taskInstance.getProcessInstanceId()) {
- TaskStateEvent nextEvent = TaskStateEvent.builder()
- .processInstanceId(workflowInstance.getId())
- .taskInstanceId(nextTaskInstance.getId())
- .type(StateEventType.WAKE_UP_TASK_GROUP)
- .build();
- stateEvents.add(nextEvent);
- } else {
- ProcessInstance processInstance =
- processService.findProcessInstanceById(nextTaskInstance.getProcessInstanceId());
- if (processInstance == null) {
- log.error("WorkflowInstance is null cannot wakeup, processInstanceId:{}",
- nextTaskInstance.getProcessInstanceId());
- return;
- }
- if (processInstance.getHost() == null || Constants.NULL.equals(processInstance.getHost())) {
- log.warn("The next WorkflowInstance: {} host is null no need to wakeup, maybe it is in failover",
- processInstance);
- return;
- }
- ILogicTaskInstanceOperator taskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory
- .getProxyClient(processInstance.getHost(), ILogicTaskInstanceOperator.class);
- taskInstanceOperator.wakeupTaskInstance(
- new TaskInstanceWakeupRequest(processInstance.getId(), nextTaskInstance.getId()));
- }
- log.info("Success send wakeup message to next taskInstance: {}", nextTaskInstance.getId());
+ taskGroupCoordinator.releaseTaskGroupSlot(taskInstance);
+ log.info("Success release task Group slot: {} for taskInstance: {} ", taskInstance.getTaskGroupId(),
+ taskInstance.getName());
}
/**
@@ -790,10 +723,11 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
} else {
listenerEventAlertManager.publishProcessFailListenerEvent(workflowInstance, projectUser);
}
- if (checkTaskQueue()) {
- // release task group
- processService.releaseAllTaskGroup(workflowInstance.getId());
- }
+ taskInstanceMap.forEach((id, taskInstance) -> {
+ if (taskInstance != null && taskInstance.getTaskGroupId() > 0) {
+ releaseTaskGroupIfNeeded(taskInstance);
+ }
+ });
// Log the workflowInstance in detail
log.info(WorkflowInstanceUtils.logWorkflowInstanceInDetails(workflowInstance));
}
@@ -1006,19 +940,11 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
// it will be wakeup when other tasks release the resource.
int taskGroupId = taskInstance.getTaskGroupId();
if (taskGroupId > 0) {
- boolean acquireTaskGroup = processService.acquireTaskGroup(taskInstance.getId(),
- taskInstance.getName(),
- taskGroupId,
- taskInstance.getProcessInstanceId(),
- taskInstance.getTaskGroupPriority());
- if (!acquireTaskGroup) {
- log.info(
- "Submitted task will not be dispatch right now because the first time to try to acquire"
- +
- " task group failed, taskInstanceName: {}, taskGroupId: {}",
- taskInstance.getName(), taskGroupId);
- return true;
- }
+ taskGroupCoordinator.acquireTaskGroupSlot(taskInstance);
+ log.info("The TaskInstance: {} use taskGroup: {} to manage the resource, will wait to notify it",
+ taskInstance,
+ taskGroupId);
+ return true;
}
// 4. submit to dispatch queue
tryToDispatchTaskInstance(taskInstance, taskExecuteRunnable);
@@ -1371,7 +1297,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
// set the task instance state to fault tolerance
existTaskInstance.setFlag(Flag.NO);
existTaskInstance.setState(TaskExecutionStatus.NEED_FAULT_TOLERANCE);
- releaseTaskGroup(existTaskInstance);
+ releaseTaskGroupIfNeeded(existTaskInstance);
validTaskMap.remove(existTaskInstance.getTaskCode());
taskInstanceDao.updateById(existTaskInstance);
@@ -2090,16 +2016,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
return recoveryNodeCodeList;
}
- private boolean checkTaskQueue() {
- AtomicBoolean result = new AtomicBoolean(false);
- taskInstanceMap.forEach((id, taskInstance) -> {
- if (taskInstance != null && taskInstance.getTaskGroupId() > 0) {
- result.set(true);
- }
- });
- return result.get();
- }
-
private boolean isNewProcessInstance() {
ProcessInstance workflowInstance = workflowExecuteContext.getWorkflowInstance();
if (Flag.YES.equals(workflowInstance.getRecovery())) {
@@ -2126,6 +2042,17 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
return taskExecuteRunnableMap;
}
+ public Optional<DefaultTaskExecuteRunnable> getTaskExecuteRunnableById(Integer taskInstanceId) {
+ if (taskInstanceId == null) {
+ throw new IllegalArgumentException("taskInstanceId can't be null");
+ }
+ TaskInstance taskInstance = taskInstanceMap.get(taskInstanceId);
+ if (taskInstance == null) {
+ return Optional.empty();
+ }
+ return Optional.ofNullable(taskExecuteRunnableMap.get(taskInstance.getTaskCode()));
+ }
+
public Map<Long, TaskInstance> getWaitToRetryTaskInstanceMap() {
return waitToRetryTaskInstanceMap;
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java
index fcd2a5f721..7caed6ba75 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableFactory.java
@@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.exception.WorkflowCreateException;
import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory;
+import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
import org.apache.dolphinscheduler.service.command.CommandService;
@@ -73,6 +74,9 @@ public class WorkflowExecuteRunnableFactory {
@Autowired
private ListenerEventAlertManager listenerEventAlertManager;
+ @Autowired
+ private TaskGroupCoordinator taskGroupCoordinator;
+
public Optional<WorkflowExecuteRunnable> createWorkflowExecuteRunnable(Command command) throws WorkflowCreateException {
try {
Optional<IWorkflowExecuteContext> workflowExecuteRunnableContextOptional =
@@ -88,9 +92,10 @@ public class WorkflowExecuteRunnableFactory {
curingGlobalParamsService,
taskInstanceDao,
defaultTaskExecuteRunnableFactory,
- listenerEventAlertManager));
+ listenerEventAlertManager,
+ taskGroupCoordinator));
} catch (Exception ex) {
- throw new WorkflowCreateException("Create workflow execute runnable failed", ex);
+ throw new WorkflowCreateException("Create WorkflowExecuteRunnable failed", ex);
}
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java
new file mode 100644
index 0000000000..fae0d2b91f
--- /dev/null
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinator.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.runner.taskgroup;
+
+import org.apache.dolphinscheduler.common.constants.Constants;
+import org.apache.dolphinscheduler.common.enums.Flag;
+import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
+import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
+import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager;
+import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
+import org.apache.dolphinscheduler.common.thread.ThreadUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
+import org.apache.dolphinscheduler.dao.entity.TaskGroup;
+import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupDao;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupQueueDao;
+import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
+import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
+import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService;
+import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
+import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupResponse;
+import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
+import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
+import org.apache.dolphinscheduler.registry.api.RegistryClient;
+import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.time.StopWatch;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * The TaskGroupCoordinator use to manage the task group slot. The task group slot is used to limit the number of {@link TaskInstance} that can be run at the same time.
+ * <p>
+ * The {@link TaskGroupQueue} is used to represent the task group slot. When a {@link TaskGroupQueue} which inQueue is YES means the {@link TaskGroupQueue} is using by a {@link TaskInstance}.
+ * <p>
+ * When the {@link TaskInstance} need to use task group, we should use @{@link TaskGroupCoordinator#acquireTaskGroupSlot(TaskInstance)} to acquire the task group slot,
+ * this method doesn't block should always acquire successfully, and you should directly stop dispatch the task instance.
+ * When the task group slot is available, the TaskGroupCoordinator will wake up the waiting {@link TaskInstance} to dispatch.
+ * <pre>
+ * if(needAcquireTaskGroupSlot(taskInstance)) {
+ * taskGroupCoordinator.acquireTaskGroupSlot(taskInstance);
+ * return;
+ * }
+ * </pre>
+ * <p>
+ * When the {@link TaskInstance} is finished, we should use @{@link TaskGroupCoordinator#releaseTaskGroupSlot(TaskInstance)} to release the task group slot.
+ * <pre>
+ * if(needToReleaseTaskGroupSlot(taskInstance)) {
+ * taskGroupCoordinator.releaseTaskGroupSlot(taskInstance);
+ * }
+ * </pre>
+ */
+@Slf4j
+@Component
+public class TaskGroupCoordinator extends BaseDaemonThread {
+
+ @Autowired
+ private RegistryClient registryClient;
+
+ @Autowired
+ private TaskGroupDao taskGroupDao;
+
+ @Autowired
+ private TaskGroupQueueDao taskGroupQueueDao;
+
+ @Autowired
+ private TaskInstanceDao taskInstanceDao;
+
+ @Autowired
+ private ProcessInstanceDao processInstanceDao;
+
+ public TaskGroupCoordinator() {
+ super("TaskGroupCoordinator");
+ }
+
+ @Override
+ public synchronized void start() {
+ log.info("TaskGroupCoordinator starting...");
+ super.start();
+ log.info("TaskGroupCoordinator started...");
+ }
+
+ @Override
+ public void run() {
+ while (!ServerLifeCycleManager.isStopped()) {
+ try {
+ if (!ServerLifeCycleManager.isRunning()) {
+ continue;
+ }
+ try {
+ registryClient.getLock(RegistryNodeType.MASTER_TASK_GROUP_COORDINATOR_LOCK.getRegistryPath());
+ StopWatch taskGroupCoordinatorRoundTimeCost = StopWatch.createStarted();
+
+ amendTaskGroupUseSize();
+ amendTaskGroupQueueStatus();
+ dealWithForceStartTaskGroupQueue();
+ dealWithWaitingTaskGroupQueue();
+
+ taskGroupCoordinatorRoundTimeCost.stop();
+ log.info("TaskGroupCoordinator round time cost: {}/ms",
+ taskGroupCoordinatorRoundTimeCost.getTime());
+ } finally {
+ registryClient.releaseLock(RegistryNodeType.MASTER_TASK_GROUP_COORDINATOR_LOCK.getRegistryPath());
+ }
+ } catch (Throwable e) {
+ log.error("TaskGroupCoordinator error", e);
+ } finally {
+ // sleep 5s
+ ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS * 5);
+ }
+ }
+ }
+
+ /**
+ * Make sure the TaskGroup useSize is equal to the TaskGroupQueue which status is {@link TaskGroupQueueStatus#ACQUIRE_SUCCESS} and forceStart is {@link org.apache.dolphinscheduler.common.enums.Flag#NO}.
+ */
+ private void amendTaskGroupUseSize() {
+ // The TaskGroup useSize should equal to the TaskGroupQueue which inQueue is YES and forceStart is NO
+ List<TaskGroup> taskGroups = taskGroupDao.queryAllTaskGroups();
+ if (CollectionUtils.isEmpty(taskGroups)) {
+ return;
+ }
+ for (TaskGroup taskGroup : taskGroups) {
+ List<TaskGroupQueue> taskGroupQueues =
+ taskGroupQueueDao.queryAcquiredTaskGroupQueueByGroupId(taskGroup.getId());
+ int actualUseSize = taskGroupQueues.size();
+ if (taskGroup.getUseSize() == actualUseSize) {
+ continue;
+ }
+ log.warn("The TaskGroup: {} useSize is {}, but the actual use size is {}, will amend it",
+ taskGroup.getName(),
+ taskGroup.getUseSize(), actualUseSize);
+ taskGroup.setUseSize(actualUseSize);
+ taskGroupDao.updateById(taskGroup);
+ }
+ }
+
+ /**
+ * Make sure the TaskGroupQueue status is {@link TaskGroupQueueStatus#RELEASE} when the related {@link TaskInstance} is not exist or status is finished.
+ */
+ private void amendTaskGroupQueueStatus() {
+ List<TaskGroupQueue> taskGroupQueues = taskGroupQueueDao.queryAllInQueueTaskGroupQueue();
+ List<Integer> taskInstanceIds = taskGroupQueues.stream()
+ .map(TaskGroupQueue::getTaskId)
+ .collect(Collectors.toList());
+ Map<Integer, TaskInstance> taskInstanceMap = taskInstanceDao.queryByIds(taskInstanceIds)
+ .stream()
+ .collect(Collectors.toMap(TaskInstance::getId, Function.identity()));
+
+ for (TaskGroupQueue taskGroupQueue : taskGroupQueues) {
+ int taskId = taskGroupQueue.getTaskId();
+ TaskInstance taskInstance = taskInstanceMap.get(taskId);
+
+ if (taskInstance == null) {
+ log.warn("The TaskInstance: {} is not exist, will release the TaskGroupQueue: {}", taskId,
+ taskGroupQueue);
+ releaseTaskGroupQueueSlot(taskGroupQueue);
+ continue;
+ }
+
+ if (taskInstance.getState().isFinished()) {
+ log.warn("The TaskInstance: {} state: {} finished, will release the TaskGroupQueue: {}",
+ taskInstance.getName(), taskInstance.getState(), taskGroupQueue);
+ releaseTaskGroupQueueSlot(taskGroupQueue);
+ continue;
+ }
+ }
+ }
+
+ private void dealWithForceStartTaskGroupQueue() {
+ // Find the force start task group queue(Which is inQueue and forceStart is YES)
+ // Notify the related waiting task instance
+ // Set the taskGroupQueue status to RELEASE and remove it from queue
+ List<TaskGroupQueue> taskGroupQueues = taskGroupQueueDao.queryAllInQueueTaskGroupQueue()
+ .stream()
+ .filter(taskGroupQueue -> Flag.YES.getCode() == taskGroupQueue.getForceStart())
+ .collect(Collectors.toList());
+ for (TaskGroupQueue taskGroupQueue : taskGroupQueues) {
+ try {
+ LogUtils.setTaskInstanceIdMDC(taskGroupQueue.getTaskId());
+ // notify the waiting task instance
+ // We notify first, it notify failed, the taskGroupQueue will be in queue, and then we will retry it
+ // next time.
+ notifyWaitingTaskInstance(taskGroupQueue);
+ log.info("Notify the ForceStart waiting TaskInstance: {} for taskGroupQueue: {} success",
+ taskGroupQueue.getTaskName(),
+ taskGroupQueue.getId());
+
+ taskGroupQueue.setInQueue(Flag.NO.getCode());
+ taskGroupQueue.setStatus(TaskGroupQueueStatus.RELEASE);
+ taskGroupQueue.setUpdateTime(new Date());
+ taskGroupQueueDao.updateById(taskGroupQueue);
+ log.info("Release the force start TaskGroupQueue {}", taskGroupQueue);
+ } catch (UnsupportedOperationException unsupportedOperationException) {
+ releaseTaskGroupQueueSlot(taskGroupQueue);
+ log.info(
+ "Notify the ForceStart TaskInstance: {} for taskGroupQueue: {} failed, will release the taskGroupQueue",
+ taskGroupQueue.getTaskName(), taskGroupQueue.getId(), unsupportedOperationException);
+ } catch (Throwable throwable) {
+ log.info("Notify the force start TaskGroupQueue {} failed", taskGroupQueue, throwable);
+ } finally {
+ LogUtils.removeTaskInstanceIdMDC();
+ }
+ }
+ }
+
+ private void dealWithWaitingTaskGroupQueue() {
+ // Find the TaskGroup which usage < maxSize.
+ // Find the highest priority inQueue task group queue(Which is inQueue and status is Waiting and force start is
+ // NO) belong to the
+ // task group.
+ List<TaskGroup> taskGroups = taskGroupDao.queryAvailableTaskGroups();
+ if (CollectionUtils.isEmpty(taskGroups)) {
+ log.debug("There is no available task group");
+ return;
+ }
+ for (TaskGroup taskGroup : taskGroups) {
+ int availableSize = taskGroup.getGroupSize() - taskGroup.getUseSize();
+ if (availableSize <= 0) {
+ log.info("TaskGroup {} is full, available size is {}", taskGroup, availableSize);
+ continue;
+ }
+ List<TaskGroupQueue> taskGroupQueues =
+ taskGroupQueueDao.queryAllInQueueTaskGroupQueueByGroupId(taskGroup.getId())
+ .stream()
+ .filter(taskGroupQueue -> Flag.NO.getCode() == taskGroupQueue.getForceStart())
+ .filter(taskGroupQueue -> TaskGroupQueueStatus.WAIT_QUEUE == taskGroupQueue.getStatus())
+ .limit(availableSize)
+ .collect(Collectors.toList());
+ if (CollectionUtils.isEmpty(taskGroupQueues)) {
+ log.debug("There is no waiting task group queue for task group {}", taskGroup.getName());
+ continue;
+ }
+ for (TaskGroupQueue taskGroupQueue : taskGroupQueues) {
+ try {
+ LogUtils.setTaskInstanceIdMDC(taskGroupQueue.getTaskId());
+ // Reduce the taskGroupSize
+ boolean acquireResult = taskGroupDao.acquireTaskGroupSlot(taskGroup.getId());
+ if (!acquireResult) {
+ log.error("Failed to acquire task group slot for task group {}", taskGroup);
+ continue;
+ }
+ // Notify the waiting task instance
+ // We notify first, it notify failed, the taskGroupQueue will be in queue, and then we will retry it
+ // next time.
+ notifyWaitingTaskInstance(taskGroupQueue);
+
+ // Set the taskGroupQueue status to RUNNING and remove from queue
+ taskGroupQueue.setInQueue(Flag.YES.getCode());
+ taskGroupQueue.setStatus(TaskGroupQueueStatus.ACQUIRE_SUCCESS);
+ taskGroupQueue.setUpdateTime(new Date());
+ taskGroupQueueDao.updateById(taskGroupQueue);
+ } catch (UnsupportedOperationException unsupportedOperationException) {
+ releaseTaskGroupQueueSlot(taskGroupQueue);
+ log.info(
+ "Notify the Waiting TaskInstance: {} for taskGroupQueue: {} failed, will release the taskGroupQueue",
+ taskGroupQueue.getTaskName(), taskGroupQueue.getId(), unsupportedOperationException);
+ } catch (Throwable throwable) {
+ log.error("Notify Waiting TaskGroupQueue: {} failed", taskGroupQueue, throwable);
+ } finally {
+ LogUtils.removeTaskInstanceIdMDC();
+ }
+ }
+ }
+ }
+
+ /**
+ * If the {@link TaskInstance#getTaskGroupId()} > 0, and the TaskGroup flag is {@link Flag#YES} then the task instance need to use task group.
+ *
+ * @param taskInstance task instance
+ * @return true if the TaskInstance need to acquireTaskGroupSlot
+ */
+ public boolean needAcquireTaskGroupSlot(TaskInstance taskInstance) {
+ if (taskInstance == null) {
+ throw new IllegalArgumentException("The TaskInstance is null");
+ }
+ if (taskInstance.getTaskGroupId() <= 0) {
+ log.debug("The current TaskInstance doesn't use TaskGroup, no need to acquire TaskGroupSlot");
+ return false;
+ }
+ TaskGroup taskGroup = taskGroupDao.queryById(taskInstance.getTaskGroupId());
+ if (taskGroup == null) {
+ log.warn("The current TaskGroup: {} does not exist, will not acquire TaskGroupSlot",
+ taskInstance.getTaskGroupId());
+ return false;
+ }
+ return Flag.YES.equals(taskGroup.getStatus());
+ }
+
+ /**
+ * Acquire the task group slot for the given {@link TaskInstance}.
+ * <p>
+ * When taskInstance want to acquire a TaskGroup slot, should call this method. If acquire successfully, will create a TaskGroupQueue in db which is in queue and status is {@link TaskGroupQueueStatus#WAIT_QUEUE}.
+ * The TaskInstance shouldn't dispatch until there exist available slot, the taskGroupCoordinator notify it.
+ *
+ * @param taskInstance the task instance which want to acquire task group slot.
+ * @throws IllegalArgumentException if the taskInstance is null or the used taskGroup doesn't exist.
+ */
+ public void acquireTaskGroupSlot(TaskInstance taskInstance) {
+ if (taskInstance == null || taskInstance.getTaskGroupId() <= 0) {
+ throw new IllegalArgumentException("The current TaskInstance does not use task group");
+ }
+ TaskGroup taskGroup = taskGroupDao.queryById(taskInstance.getTaskGroupId());
+ if (taskGroup == null) {
+ throw new IllegalArgumentException(
+ "The current TaskGroup: " + taskInstance.getTaskGroupId() + " does not exist");
+ }
+ // Write TaskGroupQueue in db, and then return wait TaskGroupCoordinator to notify it
+ // Set the taskGroupQueue status to WAIT_QUEUE and add to queue
+ // The queue only contains the taskGroupQueue which status is WAIT_QUEUE or ACQUIRE_SUCCESS
+ Date now = new Date();
+ TaskGroupQueue taskGroupQueue = TaskGroupQueue
+ .builder()
+ .taskId(taskInstance.getId())
+ .taskName(taskInstance.getName())
+ .groupId(taskInstance.getTaskGroupId())
+ .processId(taskInstance.getProcessInstanceId())
+ .priority(taskInstance.getTaskGroupPriority())
+ .inQueue(Flag.YES.getCode())
+ .forceStart(Flag.NO.getCode())
+ .status(TaskGroupQueueStatus.WAIT_QUEUE)
+ .createTime(now)
+ .updateTime(now)
+ .build();
+ log.info("Success insert TaskGroupQueue: {} for TaskInstance: {}", taskGroupQueue, taskInstance.getName());
+ taskGroupQueueDao.insert(taskGroupQueue);
+ }
+
+ /**
+ * If the TaskInstance is using TaskGroup then it need to release TaskGroupSlot.
+ *
+ * @param taskInstance taskInsatnce
+ * @return true if the TaskInstance need to release TaskGroupSlot
+ */
+ public boolean needToReleaseTaskGroupSlot(TaskInstance taskInstance) {
+ if (taskInstance == null) {
+ throw new IllegalArgumentException("The TaskInstance is null");
+ }
+ if (taskInstance.getTaskGroupId() <= 0) {
+ log.debug("The current TaskInstance doesn't use TaskGroup, no need to release TaskGroupSlot");
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Release the task group slot for the given {@link TaskInstance}.
+ * <p>
+ * When taskInstance want to release a TaskGroup slot, should call this method. The release method will move the TaskGroupQueue out queue and set status to {@link TaskGroupQueueStatus#RELEASE}.
+ * This method is idempotent, this means that if the task group slot is already released, this method will do nothing.
+ *
+ * @param taskInstance the task instance which want to release task group slot.
+ * @throws IllegalArgumentException If the taskInstance is null or the task doesn't use task group.
+ */
+ public void releaseTaskGroupSlot(TaskInstance taskInstance) {
+ if (taskInstance == null || taskInstance.getTaskGroupId() <= 0) {
+ throw new IllegalArgumentException("The current TaskInstance does not use task group");
+ }
+ List<TaskGroupQueue> taskGroupQueues = taskGroupQueueDao.queryByTaskInstanceId(taskInstance.getId());
+ for (TaskGroupQueue taskGroupQueue : taskGroupQueues) {
+ releaseTaskGroupQueueSlot(taskGroupQueue);
+ }
+ }
+
+ private void notifyWaitingTaskInstance(TaskGroupQueue taskGroupQueue) {
+ // Find the related waiting task instance
+ // send RPC to notify the waiting task instance
+ TaskInstance taskInstance = taskInstanceDao.queryById(taskGroupQueue.getTaskId());
+ if (taskInstance == null) {
+ throw new UnsupportedOperationException(
+ "The TaskInstance: " + taskGroupQueue.getTaskId() + " is not exist, no need to notify");
+ }
+ // todo: We may need to add a new status to represent the task instance is waiting for task group slot
+ if (taskInstance.getState() != TaskExecutionStatus.SUBMITTED_SUCCESS) {
+ throw new UnsupportedOperationException(
+ "The TaskInstance: " + taskInstance.getId() + " state is " + taskInstance.getState()
+ + ", no need to notify");
+ }
+ ProcessInstance processInstance = processInstanceDao.queryById(taskInstance.getProcessInstanceId());
+ if (processInstance == null) {
+ throw new UnsupportedOperationException(
+ "The WorkflowInstance: " + taskInstance.getProcessInstanceId()
+ + " is not exist, no need to notify");
+ }
+ if (processInstance.getState() != WorkflowExecutionStatus.RUNNING_EXECUTION) {
+ throw new UnsupportedOperationException(
+ "The WorkflowInstance: " + processInstance.getId() + " state is " + processInstance.getState()
+ + ", no need to notify");
+ }
+ if (processInstance.getHost() == null || Constants.NULL.equals(processInstance.getHost())) {
+ throw new UnsupportedOperationException(
+ "WorkflowInstance host is null, maybe it is in failover: " + processInstance);
+ }
+
+ TaskInstanceWakeupRequest taskInstanceWakeupRequest = TaskInstanceWakeupRequest.builder()
+ .processInstanceId(processInstance.getId())
+ .taskInstanceId(taskInstance.getId())
+ .build();
+
+ IWorkflowInstanceService iWorkflowInstanceService = SingletonJdkDynamicRpcClientProxyFactory
+ .getProxyClient(processInstance.getHost(), IWorkflowInstanceService.class);
+ TaskInstanceWakeupResponse taskInstanceWakeupResponse =
+ iWorkflowInstanceService.wakeupTaskInstance(taskInstanceWakeupRequest);
+ if (!taskInstanceWakeupResponse.isSuccess()) {
+ throw new UnsupportedOperationException(
+ "Notify TaskInstance: " + taskInstance.getId() + " failed: " + taskInstanceWakeupResponse);
+ }
+ log.info("Wake up TaskInstance: {} success", taskInstance.getName());
+ }
+
+ private void releaseTaskGroupQueueSlot(TaskGroupQueue taskGroupQueue) {
+ if (TaskGroupQueueStatus.RELEASE.equals(taskGroupQueue.getStatus())
+ && Flag.NO.getCode() == taskGroupQueue.getInQueue()) {
+ log.info("The TaskGroupQueue: {} is already released", taskGroupQueue);
+ return;
+ }
+ taskGroupQueue.setInQueue(Flag.NO.getCode());
+ taskGroupQueue.setStatus(TaskGroupQueueStatus.RELEASE);
+ taskGroupQueue.setUpdateTime(new Date());
+ taskGroupQueueDao.updateById(taskGroupQueue);
+ log.info("Success release TaskGroupQueue: {}", taskGroupQueue);
+ }
+
+}
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
index 28c0010e3b..c08fb206f4 100644
--- a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnableTest.java
@@ -39,6 +39,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.graph.IWorkflowGraph;
import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory;
+import org.apache.dolphinscheduler.server.master.runner.taskgroup.TaskGroupCoordinator;
import org.apache.dolphinscheduler.service.alert.ListenerEventAlertManager;
import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
@@ -49,7 +50,6 @@ import org.apache.dolphinscheduler.service.process.ProcessService;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
-import java.text.ParseException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
@@ -104,6 +104,8 @@ public class WorkflowExecuteRunnableTest {
private ListenerEventAlertManager listenerEventAlertManager;
+ private TaskGroupCoordinator taskGroupCoordinator;
+
@BeforeEach
public void init() throws Exception {
applicationContext = Mockito.mock(ApplicationContext.class);
@@ -138,6 +140,8 @@ public class WorkflowExecuteRunnableTest {
Mockito.when(workflowExecuteContext.getWorkflowGraph()).thenReturn(workflowGraph);
Mockito.when(workflowGraph.getDag()).thenReturn(new DAG<>());
+ taskGroupCoordinator = Mockito.mock(TaskGroupCoordinator.class);
+
workflowExecuteThread = Mockito.spy(
new WorkflowExecuteRunnable(
workflowExecuteContext,
@@ -150,11 +154,12 @@ public class WorkflowExecuteRunnableTest {
curingGlobalParamsService,
taskInstanceDao,
defaultTaskExecuteRunnableFactory,
- listenerEventAlertManager));
+ listenerEventAlertManager,
+ taskGroupCoordinator));
}
@Test
- public void testParseStartNodeName() throws ParseException {
+ public void testParseStartNodeName() {
try {
Map<String, String> cmdParam = new HashMap<>();
cmdParam.put(CMD_PARAM_START_NODES, "1,2,3");
diff --git a/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinatorTest.java b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinatorTest.java
new file mode 100644
index 0000000000..f654550b75
--- /dev/null
+++ b/dolphinscheduler-master/src/test/java/org/apache/dolphinscheduler/server/master/runner/taskgroup/TaskGroupCoordinatorTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.master.runner.taskgroup;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.dolphinscheduler.common.enums.Flag;
+import org.apache.dolphinscheduler.common.enums.TaskGroupQueueStatus;
+import org.apache.dolphinscheduler.dao.entity.TaskGroup;
+import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
+import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupDao;
+import org.apache.dolphinscheduler.dao.repository.TaskGroupQueueDao;
+import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
+import org.apache.dolphinscheduler.registry.api.RegistryClient;
+import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
+
+import com.google.common.collect.Lists;
+
+@ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.LENIENT)
+class TaskGroupCoordinatorTest {
+
+ @InjectMocks
+ private TaskGroupCoordinator taskGroupCoordinator;
+
+ @Mock
+ private RegistryClient registryClient;
+
+ @Mock
+ private TaskGroupDao taskGroupDao;
+
+ @Mock
+ private TaskGroupQueueDao taskGroupQueueDao;
+
+ @Mock
+ private TaskInstanceDao taskInstanceDao;
+
+ @Mock
+ private ProcessInstanceDao processInstanceDao;
+
+ @Test
+ void start() throws InterruptedException {
+ // Get the Lock from Registry
+ taskGroupCoordinator.start();
+ Thread.sleep(1_000);
+ verify(registryClient, Mockito.times(1))
+ .getLock(RegistryNodeType.MASTER_TASK_GROUP_COORDINATOR_LOCK.getRegistryPath());
+ verify(registryClient, Mockito.times(1))
+ .releaseLock(RegistryNodeType.MASTER_TASK_GROUP_COORDINATOR_LOCK.getRegistryPath());
+
+ }
+
+ @Test
+ void needAcquireTaskGroupSlot() {
+ // TaskInstance is null
+ IllegalArgumentException illegalArgumentException =
+ assertThrows(IllegalArgumentException.class, () -> taskGroupCoordinator.needAcquireTaskGroupSlot(null));
+ assertEquals("The TaskInstance is null", illegalArgumentException.getMessage());
+
+ // TaskGroupId < 0
+ TaskInstance taskInstance = new TaskInstance();
+ assertFalse(taskGroupCoordinator.needAcquireTaskGroupSlot(taskInstance));
+
+ // TaskGroup not exist
+ taskInstance.setTaskGroupId(1);
+ when(taskGroupDao.queryById(taskInstance.getTaskGroupId())).thenReturn(null);
+ assertFalse(taskGroupCoordinator.needAcquireTaskGroupSlot(taskInstance));
+
+ // TaskGroup is closed
+ TaskGroup taskGroup = new TaskGroup();
+ taskGroup.setStatus(Flag.NO);
+ when(taskGroupDao.queryById(taskInstance.getTaskGroupId())).thenReturn(taskGroup);
+ assertFalse(taskGroupCoordinator.needAcquireTaskGroupSlot(taskInstance));
+
+ // TaskGroup is open
+ taskGroup.setStatus(Flag.YES);
+ when(taskGroupDao.queryById(taskInstance.getTaskGroupId())).thenReturn(taskGroup);
+ assertTrue(taskGroupCoordinator.needToReleaseTaskGroupSlot(taskInstance));
+
+ }
+
+ @Test
+ void acquireTaskGroupSlot() {
+ // TaskInstance is NULL
+ IllegalArgumentException illegalArgumentException =
+ assertThrows(IllegalArgumentException.class, () -> taskGroupCoordinator.acquireTaskGroupSlot(null));
+ assertEquals("The current TaskInstance does not use task group", illegalArgumentException.getMessage());
+
+ // TaskGroupId is NULL
+ TaskInstance taskInstance = new TaskInstance();
+ illegalArgumentException = assertThrows(IllegalArgumentException.class,
+ () -> taskGroupCoordinator.acquireTaskGroupSlot(taskInstance));
+ assertEquals("The current TaskInstance does not use task group", illegalArgumentException.getMessage());
+
+ // TaskGroup not exist
+ taskInstance.setTaskGroupId(1);
+ taskInstance.setId(1);
+ when(taskGroupDao.queryById(taskInstance.getTaskGroupId())).thenReturn(null);
+ illegalArgumentException = assertThrows(IllegalArgumentException.class,
+ () -> taskGroupCoordinator.acquireTaskGroupSlot(taskInstance));
+ assertEquals("The current TaskGroup: 1 does not exist", illegalArgumentException.getMessage());
+
+ // TaskGroup exist
+ when(taskGroupDao.queryById(taskInstance.getTaskGroupId())).thenReturn(new TaskGroup());
+ Assertions.assertDoesNotThrow(() -> taskGroupCoordinator.acquireTaskGroupSlot(taskInstance));
+
+ }
+
+ @Test
+ void needToReleaseTaskGroupSlot() {
+ IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class,
+ () -> taskGroupCoordinator.needToReleaseTaskGroupSlot(null));
+ assertEquals("The TaskInstance is null", illegalArgumentException.getMessage());
+
+ TaskInstance taskInstance = new TaskInstance();
+ assertFalse(taskGroupCoordinator.needToReleaseTaskGroupSlot(taskInstance));
+
+ taskInstance.setTaskGroupId(1);
+ assertTrue(taskGroupCoordinator.needToReleaseTaskGroupSlot(taskInstance));
+ }
+
+ @Test
+ void releaseTaskGroupSlot() {
+ // TaskInstance is NULL
+ IllegalArgumentException illegalArgumentException =
+ assertThrows(IllegalArgumentException.class, () -> taskGroupCoordinator.releaseTaskGroupSlot(null));
+ assertEquals("The current TaskInstance does not use task group", illegalArgumentException.getMessage());
+
+ // TaskGroupId is NULL
+ TaskInstance taskInstance = new TaskInstance();
+ illegalArgumentException = assertThrows(IllegalArgumentException.class,
+ () -> taskGroupCoordinator.releaseTaskGroupSlot(taskInstance));
+ assertEquals("The current TaskInstance does not use task group", illegalArgumentException.getMessage());
+
+ // Release TaskGroupQueue
+ taskInstance.setId(1);
+ taskInstance.setTaskGroupId(1);
+ TaskGroupQueue taskGroupQueue = new TaskGroupQueue();
+ List<TaskGroupQueue> taskGroupQueues = Lists.newArrayList(taskGroupQueue);
+ when(taskGroupQueueDao.queryByTaskInstanceId(taskInstance.getId())).thenReturn(taskGroupQueues);
+ taskGroupCoordinator.releaseTaskGroupSlot(taskInstance);
+
+ assertEquals(Flag.NO.getCode(), taskGroupQueue.getInQueue());
+ assertEquals(TaskGroupQueueStatus.RELEASE, taskGroupQueue.getStatus());
+ verify(taskGroupQueueDao, Mockito.times(1)).updateById(taskGroupQueue);
+
+ }
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java
index f5c07e66f6..a1f3bb02b7 100644
--- a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/enums/RegistryNodeType.java
@@ -28,6 +28,7 @@ public enum RegistryNodeType {
MASTER("Master", "/nodes/master"),
MASTER_NODE_LOCK("MasterNodeLock", "/lock/master-node"),
MASTER_FAILOVER_LOCK("MasterFailoverLock", "/lock/master-failover"),
+ MASTER_TASK_GROUP_COORDINATOR_LOCK("TaskGroupCoordinatorLock", "/lock/master-task-group-coordinator"),
WORKER("Worker", "/nodes/worker"),
ALERT_SERVER("AlertServer", "/nodes/alert-server"),
ALERT_LOCK("AlertNodeLock", "/lock/alert"),
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index 70d2593a32..091d6353fb 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -37,7 +37,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
-import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
@@ -51,7 +50,6 @@ import org.apache.dolphinscheduler.service.exceptions.CronParseException;
import org.apache.dolphinscheduler.service.model.TaskNode;
import java.util.List;
-import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;
@@ -72,8 +70,6 @@ public interface ProcessService {
ProcessInstance findProcessInstanceById(int processId);
- ProcessDefinition findProcessDefineById(int processDefinitionId);
-
ProcessDefinition findProcessDefinition(Long processDefinitionCode, int processDefinitionVersion);
ProcessDefinition findProcessDefinitionByCode(Long processDefinitionCode);
@@ -92,9 +88,6 @@ public interface ProcessService {
void setSubProcessParam(ProcessInstance subProcessInstance);
- boolean submitTaskWithRetry(ProcessInstance processInstance, TaskInstance taskInstance, int commitRetryTimes,
- long commitInterval);
-
@Transactional
boolean submitTask(ProcessInstance processInstance, TaskInstance taskInstance);
@@ -129,18 +122,10 @@ public interface ProcessService {
DataSource findDataSourceById(int id);
- ProcessInstance findProcessInstanceByTaskId(int taskId);
-
List<UdfFunc> queryUdfFunListByIds(Integer[] ids);
- List<Schedule> selectAllByProcessDefineCode(long[] codes);
-
- String queryUserQueueByProcessInstance(ProcessInstance processInstance);
-
ProjectUser queryProjectWithUserByProcessInstanceId(int processInstanceId);
- List<Project> getProjectListHavePerm(int userId);
-
<T> List<T> listUnauthorized(int userId, T[] needChecks, AuthorizationType authorizationType);
User getUserById(int userId);
@@ -175,8 +160,6 @@ public interface ProcessService {
List<TaskNode> transformTask(List<ProcessTaskRelation> taskRelationList,
List<TaskDefinitionLog> taskDefinitionLogs);
- Map<ProcessInstance, TaskInstance> notifyProcessList(int processId);
-
DqExecuteResult getDqExecuteResultByTaskInstanceId(int taskInstanceId);
int updateDqExecuteResultUserId(int taskInstanceId);
@@ -195,16 +178,6 @@ public interface ProcessService {
DqComparisonType getComparisonTypeById(int id);
- boolean acquireTaskGroup(int taskId,
- String taskName, int groupId,
- int processId, int priority);
-
- boolean robTaskGroupResource(TaskGroupQueue taskGroupQueue);
-
- void releaseAllTaskGroup(int processInstanceId);
-
- TaskInstance releaseTaskGroup(TaskInstance taskInstance);
-
void changeTaskGroupQueueStatus(int taskId, TaskGroupQueueStatus status);
TaskGroupQueue insertIntoTaskGroupQueue(Integer taskId,
@@ -214,12 +187,6 @@ public interface ProcessService {
Integer priority,
TaskGroupQueueStatus status);
- int updateTaskGroupQueueStatus(Integer taskId, int status);
-
- int updateTaskGroupQueue(TaskGroupQueue taskGroupQueue);
-
- TaskGroupQueue loadTaskGroupQueue(int taskId);
-
ProcessInstance loadNextProcess4Serial(long code, int state, int id);
public String findConfigYamlByName(String clusterName);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 706e183cf0..93b257ba27 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -67,12 +67,10 @@ import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
-import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
-import org.apache.dolphinscheduler.dao.entity.TaskGroup;
import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
@@ -410,17 +408,6 @@ public class ProcessServiceImpl implements ProcessService {
return processInstanceMapper.selectById(processId);
}
- /**
- * find process define by id.
- *
- * @param processDefinitionId processDefinitionId
- * @return process definition
- */
- @Override
- public ProcessDefinition findProcessDefineById(int processDefinitionId) {
- return processDefineMapper.selectById(processDefinitionId);
- }
-
/**
* find process define by code and version.
*
@@ -1080,34 +1067,6 @@ public class ProcessServiceImpl implements ProcessService {
taskInstanceDao.updateById(taskInstance);
}
- /**
- * retry submit task to db
- */
- @Override
- public boolean submitTaskWithRetry(ProcessInstance processInstance, TaskInstance taskInstance,
- int commitRetryTimes, long commitInterval) {
- int retryTimes = 1;
- while (retryTimes <= commitRetryTimes) {
- try {
- // submit task to db
- // Only want to use transaction here
- if (submitTask(processInstance, taskInstance)) {
- return true;
- }
- log.error(
- "task commit to db failed , taskCode: {} has already retry {} times, please check the database",
- taskInstance.getTaskCode(),
- retryTimes);
- Thread.sleep(commitInterval);
- } catch (Exception e) {
- log.error("task commit to db failed", e);
- } finally {
- retryTimes += 1;
- }
- }
- return false;
- }
-
/**
* // todo: This method need to refactor, we find when the db down, but the taskInstanceId is not 0. It's better to change to void, rather than return TaskInstance
* submit task to db
@@ -1260,20 +1219,6 @@ public class ProcessServiceImpl implements ProcessService {
}
}
- /**
- * get sub work flow command type
- * child instance exist: child command = fatherCommand
- * child instance not exists: child command = fatherCommand[0]
- */
- private CommandType getSubCommandType(ProcessInstance parentProcessInstance, ProcessInstance childInstance) {
- CommandType commandType = parentProcessInstance.getCommandType();
- if (childInstance == null) {
- String fatherHistoryCommand = parentProcessInstance.getHistoryCmd();
- commandType = CommandType.valueOf(fatherHistoryCommand.split(Constants.COMMA)[0]);
- }
- return commandType;
- }
-
/**
* update sub process definition
*
@@ -1573,21 +1518,6 @@ public class ProcessServiceImpl implements ProcessService {
return dataSourceMapper.selectById(id);
}
- /**
- * find process instance by the task id
- *
- * @param taskId taskId
- * @return process instance
- */
- @Override
- public ProcessInstance findProcessInstanceByTaskId(int taskId) {
- TaskInstance taskInstance = taskInstanceMapper.selectById(taskId);
- if (taskInstance != null) {
- return processInstanceMapper.selectById(taskInstance.getProcessInstanceId());
- }
- return null;
- }
-
/**
* find udf function list by id list string
*
@@ -1599,37 +1529,6 @@ public class ProcessServiceImpl implements ProcessService {
return udfFuncMapper.queryUdfByIdStr(ids, null);
}
- /**
- * find schedule list by process define codes.
- *
- * @param codes codes
- * @return schedule list
- */
- @Override
- public List<Schedule> selectAllByProcessDefineCode(long[] codes) {
- return scheduleMapper.selectAllByProcessDefineArray(codes);
- }
-
- /**
- * query user queue by process instance
- *
- * @param processInstance processInstance
- * @return queue
- */
- @Override
- public String queryUserQueueByProcessInstance(ProcessInstance processInstance) {
-
- String queue = "";
- if (processInstance == null) {
- return queue;
- }
- User executor = userMapper.selectById(processInstance.getExecutorId());
- if (executor != null) {
- queue = executor.getQueue();
- }
- return queue;
- }
-
/**
* query project name and user name by processInstanceId.
*
@@ -1641,27 +1540,6 @@ public class ProcessServiceImpl implements ProcessService {
return projectMapper.queryProjectWithUserByProcessInstanceId(processInstanceId);
}
- /**
- * get have perm project list
- *
- * @param userId userId
- * @return project list
- */
- @Override
- public List<Project> getProjectListHavePerm(int userId) {
- List<Project> createProjects = projectMapper.queryProjectCreatedByUser(userId);
- List<Project> authedProjects = projectMapper.queryAuthedProjectListByUserId(userId);
-
- if (createProjects == null) {
- createProjects = new ArrayList<>();
- }
-
- if (authedProjects != null) {
- createProjects.addAll(authedProjects);
- }
- return createProjects;
- }
-
/**
* list unauthorized udf function
*
@@ -2108,23 +1986,6 @@ public class ProcessServiceImpl implements ProcessService {
return taskNodeList;
}
- @Override
- public Map<ProcessInstance, TaskInstance> notifyProcessList(int processId) {
- HashMap<ProcessInstance, TaskInstance> processTaskMap = new HashMap<>();
- // find sub tasks
- ProcessInstanceMap processInstanceMap = processInstanceMapMapper.queryBySubProcessId(processId);
- if (processInstanceMap == null) {
- return processTaskMap;
- }
- ProcessInstance fatherProcess = this.findProcessInstanceById(processInstanceMap.getParentProcessInstanceId());
- TaskInstance fatherTask = taskInstanceDao.queryById(processInstanceMap.getParentTaskInstanceId());
-
- if (fatherProcess != null) {
- processTaskMap.put(fatherProcess, fatherTask);
- }
- return processTaskMap;
- }
-
@Override
public DqExecuteResult getDqExecuteResultByTaskInstanceId(int taskInstanceId) {
return dqExecuteResultMapper.getExecuteResultById(taskInstanceId);
@@ -2195,167 +2056,6 @@ public class ProcessServiceImpl implements ProcessService {
return dqComparisonTypeMapper.selectById(id);
}
- /**
- * the first time (when submit the task ) get the resource of the task group
- */
- @Override
- public boolean acquireTaskGroup(int taskInstanceId,
- String taskName,
- int taskGroupId,
- int workflowInstanceId,
- int taskGroupPriority) {
- TaskGroup taskGroup = taskGroupMapper.selectById(taskGroupId);
- if (taskGroup == null) {
- // we don't throw exception here, to avoid the task group has been deleted during workflow running
- log.warn("The taskGroup is not exist no need to acquire taskGroup, taskGroupId: {}", taskGroupId);
- return true;
- }
- // if task group is not applicable
- if (taskGroup.getStatus() == Flag.NO.getCode()) {
- log.warn("The taskGroup status is {}, no need to acquire taskGroup, taskGroupId: {}", taskGroup.getStatus(),
- taskGroupId);
- return true;
- }
- // Create a waiting taskGroupQueue, after acquire resource, we can update the status to ACQUIRE_SUCCESS
- TaskGroupQueue taskGroupQueue = taskGroupQueueMapper.queryByTaskId(taskInstanceId);
- if (taskGroupQueue == null) {
- taskGroupQueue = insertIntoTaskGroupQueue(
- taskInstanceId,
- taskName,
- taskGroupId,
- workflowInstanceId,
- taskGroupPriority,
- TaskGroupQueueStatus.WAIT_QUEUE);
- log.info("Insert TaskGroupQueue: {} successfully", taskGroupQueue.getId());
- } else {
- log.info("The task queue is already exist, taskId: {}", taskInstanceId);
- if (taskGroupQueue.getStatus() == TaskGroupQueueStatus.ACQUIRE_SUCCESS) {
- return true;
- }
- }
- // check if there already exist higher priority tasks
- List<TaskGroupQueue> highPriorityTasks = taskGroupQueueMapper.queryHighPriorityTasks(
- taskGroupId,
- taskGroupPriority,
- TaskGroupQueueStatus.WAIT_QUEUE.getCode());
- if (CollectionUtils.isNotEmpty(highPriorityTasks)) {
- return false;
- }
- // try to get taskGroup
- int availableTaskGroupCount = taskGroupMapper.selectAvailableCountById(taskGroupId);
- if (availableTaskGroupCount < 1) {
- log.info(
- "Failed to acquire taskGroup, there is no avaliable taskGroup, taskInstanceId: {}, taskGroupId: {}",
- taskInstanceId, taskGroupId);
- taskGroupQueueMapper.updateInQueue(Flag.NO.getCode(), taskGroupQueue.getId());
- return false;
- }
- return robTaskGroupResource(taskGroupQueue);
- }
-
- /**
- * try to get the task group resource(when other task release the resource)
- */
- @Override
- public boolean robTaskGroupResource(TaskGroupQueue taskGroupQueue) {
- // set the default max size to avoid dead loop
- for (int i = 0; i < 10; i++) {
- TaskGroup taskGroup = taskGroupMapper.selectById(taskGroupQueue.getGroupId());
- if (taskGroup.getGroupSize() <= taskGroup.getUseSize()) {
- // remove
- taskGroupQueueMapper.updateInQueue(Flag.NO.getCode(), taskGroupQueue.getId());
- log.info("The current task Group is full, taskGroup: {}", taskGroup);
- return false;
- }
- int affectedCount = taskGroupMapper.robTaskGroupResource(
- taskGroup.getId(),
- taskGroup.getUseSize(),
- taskGroupQueue.getId(),
- TaskGroupQueueStatus.WAIT_QUEUE.getCode());
- if (affectedCount > 0) {
- log.info("Success rob taskGroup, taskInstanceId: {}, taskGroupId: {}", taskGroupQueue.getTaskId(),
- taskGroupQueue.getId());
- taskGroupQueue.setStatus(TaskGroupQueueStatus.ACQUIRE_SUCCESS);
- this.taskGroupQueueMapper.updateById(taskGroupQueue);
- this.taskGroupQueueMapper.updateInQueue(Flag.NO.getCode(), taskGroupQueue.getId());
- return true;
- }
- }
- log.info("Failed to rob taskGroup, taskGroupQueue: {}", taskGroupQueue);
- taskGroupQueueMapper.updateInQueue(Flag.NO.getCode(), taskGroupQueue.getId());
- return false;
- }
-
- @Override
- public void releaseAllTaskGroup(int processInstanceId) {
- List<TaskInstance> taskInstances = this.taskInstanceMapper.loadAllInfosNoRelease(processInstanceId,
- TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode());
- for (TaskInstance info : taskInstances) {
- releaseTaskGroup(info);
- }
- }
-
- /**
- * release the TGQ resource when the corresponding task is finished.
- *
- * @return the result code and msg
- */
- @Override
- public TaskInstance releaseTaskGroup(TaskInstance taskInstance) {
-
- TaskGroup taskGroup;
- TaskGroupQueue thisTaskGroupQueue;
- log.info("Begin to release task group: {}", taskInstance.getTaskGroupId());
- try {
- do {
- taskGroup = taskGroupMapper.selectById(taskInstance.getTaskGroupId());
- if (taskGroup == null) {
- log.error("The taskGroup is not exist no need to release taskGroup, taskGroupId: {}",
- taskInstance.getTaskGroupId());
- return null;
- }
- thisTaskGroupQueue = taskGroupQueueMapper.queryByTaskId(taskInstance.getId());
- if (thisTaskGroupQueue.getStatus() == TaskGroupQueueStatus.RELEASE) {
- log.info("The taskGroupQueue's status is release, taskInstanceId: {}", taskInstance.getId());
- return null;
- }
- if (thisTaskGroupQueue.getStatus() == TaskGroupQueueStatus.WAIT_QUEUE) {
- log.info("The taskGroupQueue's status is in waiting, will not need to release task group");
- break;
- }
- } while (thisTaskGroupQueue.getForceStart() == Flag.NO.getCode()
- && taskGroupMapper.releaseTaskGroupResource(taskGroup.getId(),
- taskGroup.getUseSize(),
- thisTaskGroupQueue.getId(),
- TaskGroupQueueStatus.ACQUIRE_SUCCESS.getCode()) != 1);
- } catch (Exception e) {
- log.error("release the task group error", e);
- return null;
- }
- log.info("Finished to release task group, taskGroupId: {}", taskInstance.getTaskGroupId());
-
- log.info("Begin to release task group queue, taskGroupId: {}", taskInstance.getTaskGroupId());
- changeTaskGroupQueueStatus(taskInstance.getId(), TaskGroupQueueStatus.RELEASE);
- TaskGroupQueue taskGroupQueue;
- do {
- taskGroupQueue = taskGroupQueueMapper.queryTheHighestPriorityTasks(
- taskGroup.getId(),
- TaskGroupQueueStatus.WAIT_QUEUE.getCode(),
- Flag.NO.getCode(),
- Flag.NO.getCode());
- if (taskGroupQueue == null) {
- log.info("There is no taskGroupQueue need to be wakeup taskGroup: {}", taskGroup.getId());
- return null;
- }
- } while (this.taskGroupQueueMapper.updateInQueueCAS(
- Flag.NO.getCode(),
- Flag.YES.getCode(),
- taskGroupQueue.getId()) != 1);
- log.info("Finished to release task group queue: taskGroupId: {}, taskGroupQueueId: {}",
- taskInstance.getTaskGroupId(), taskGroupQueue.getId());
- return taskInstanceMapper.selectById(taskGroupQueue.getTaskId());
- }
-
/**
* release the TGQ resource when the corresponding task is finished.
*
@@ -2396,21 +2096,6 @@ public class ProcessServiceImpl implements ProcessService {
return taskGroupQueue;
}
- @Override
- public int updateTaskGroupQueueStatus(Integer taskId, int status) {
- return taskGroupQueueMapper.updateStatusByTaskId(taskId, status);
- }
-
- @Override
- public int updateTaskGroupQueue(TaskGroupQueue taskGroupQueue) {
- return taskGroupQueueMapper.updateById(taskGroupQueue);
- }
-
- @Override
- public TaskGroupQueue loadTaskGroupQueue(int taskId) {
- return this.taskGroupQueueMapper.queryByTaskId(taskId);
- }
-
@Override
public ProcessInstance loadNextProcess4Serial(long code, int state, int id) {
return this.processInstanceMapper.loadNextProcess4Serial(code, state, id);
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
index 70157f6a05..5e20c5ea2b 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
@@ -749,22 +749,6 @@ public class ProcessServiceTest {
Assertions.assertNotNull(taskGroupQueue);
}
- @Test
- public void testDoRelease() {
-
- TaskGroupQueue taskGroupQueue = getTaskGroupQueue();
- TaskInstance taskInstance = new TaskInstance();
- taskInstance.setId(1);
- taskInstance.setProcessInstanceId(1);
- taskInstance.setTaskGroupId(taskGroupQueue.getGroupId());
-
- when(taskGroupQueueMapper.queryByTaskId(1)).thenReturn(taskGroupQueue);
- when(taskGroupQueueMapper.updateById(taskGroupQueue)).thenReturn(1);
-
- processService.releaseTaskGroup(taskInstance);
-
- }
-
private TaskGroupQueue getTaskGroupQueue() {
TaskGroupQueue taskGroupQueue = new TaskGroupQueue();
taskGroupQueue.setTaskName("task name");
(dolphinscheduler) 15/18: Fix create parent directory will cause FileAlreadyExistsException (#15576)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit eff7364ee1b58e17abb8526950561915cf3dda0c
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Feb 6 11:59:07 2024 +0800
Fix create parent directory will cause FileAlreadyExistsException (#15576)
Co-authored-by: Rick Cheng <ri...@gmail.com>
(cherry picked from commit 4541e74e531eccc4141865cf44433753a4ed0b17)
---
.../org/apache/dolphinscheduler/common/utils/FileUtils.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index 1f3ccfed3d..fee1d9a95c 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -35,6 +35,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.PosixFilePermission;
@@ -287,8 +288,13 @@ public class FileUtils {
createDirectoryWith755(parent);
}
- Files.createDirectory(path);
- Files.setPosixFilePermissions(path, PERMISSION_755);
+ try {
+ Files.createDirectory(path);
+ Files.setPosixFilePermissions(path, PERMISSION_755);
+ } catch (FileAlreadyExistsException fileAlreadyExistsException) {
+ // Catch the FileAlreadyExistsException here to avoid create the same parent directory in parallel
+ log.debug("The directory: {} already exists", path);
+ }
}
}
(dolphinscheduler) 11/18: fix: start param for wf not work (#15544)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit fa7b4c1c8fa57a9379f4c768ab945d07ff908a1f
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Mon Feb 5 16:17:30 2024 +0800
fix: start param for wf not work (#15544)
* fix: start param for wf not work
fix: #15280
* fix test
(cherry picked from commit 01eb8f834ff4abf662b704379f1bcd019f4d2d74)
---
.../master/runner/WorkflowExecuteRunnable.java | 39 +------------------
.../service/expand/CuringParamsService.java | 12 ++++++
.../service/expand/CuringParamsServiceImpl.java | 27 ++++++++++++-
.../service/process/ProcessService.java | 3 ++
.../service/process/ProcessServiceImpl.java | 31 +++++++--------
.../service/expand/CuringParamsServiceTest.java | 44 ++++++++++++++++++++++
6 files changed, 99 insertions(+), 57 deletions(-)
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index 2db49c6f09..fa658c04f1 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.server.master.runner;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_COMPLEMENT_DATA_END_DATE;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_COMPLEMENT_DATA_SCHEDULE_DATE_LIST;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_COMPLEMENT_DATA_START_DATE;
-import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_FATHER_PARAMS;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_RECOVERY_START_NODE_STRING;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_START_NODES;
@@ -29,8 +28,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.COMMA;
import static org.apache.dolphinscheduler.common.constants.Constants.DEFAULT_WORKER_GROUP;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYY_MM_DD_HH_MM_SS;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_BLOCKING;
-import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR;
-import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -860,7 +857,7 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
Map<String, String> cmdParam = JSONUtils.toMap(workflowInstance.getCommandParam());
if (cmdParam != null) {
// reset global params while there are start parameters
- setGlobalParamIfCommanded(workflowDefinition, cmdParam);
+ processService.setGlobalParamIfCommanded(workflowDefinition, cmdParam);
Date start = null;
Date end = null;
@@ -2057,40 +2054,6 @@ public class WorkflowExecuteRunnable implements IWorkflowExecuteRunnable {
return waitToRetryTaskInstanceMap;
}
- private void setGlobalParamIfCommanded(ProcessDefinition processDefinition, Map<String, String> cmdParam) {
- // get start params from command param
- Map<String, String> startParamMap = new HashMap<>();
- if (cmdParam.containsKey(CMD_PARAM_START_PARAMS)) {
- String startParamJson = cmdParam.get(CMD_PARAM_START_PARAMS);
- startParamMap = JSONUtils.toMap(startParamJson);
- }
- Map<String, String> fatherParamMap = new HashMap<>();
- if (cmdParam.containsKey(CMD_PARAM_FATHER_PARAMS)) {
- String fatherParamJson = cmdParam.get(CMD_PARAM_FATHER_PARAMS);
- fatherParamMap = JSONUtils.toMap(fatherParamJson);
- }
- startParamMap.putAll(fatherParamMap);
- // set start param into global params
- Map<String, String> globalMap = processDefinition.getGlobalParamMap();
- List<Property> globalParamList = processDefinition.getGlobalParamList();
- if (startParamMap.size() > 0 && globalMap != null) {
- // start param to overwrite global param
- for (Map.Entry<String, String> param : globalMap.entrySet()) {
- String val = startParamMap.get(param.getKey());
- if (val != null) {
- param.setValue(val);
- }
- }
- // start param to create new global param if global not exist
- for (Map.Entry<String, String> startParam : startParamMap.entrySet()) {
- if (!globalMap.containsKey(startParam.getKey())) {
- globalMap.put(startParam.getKey(), startParam.getValue());
- globalParamList.add(new Property(startParam.getKey(), IN, VARCHAR, startParam.getValue()));
- }
- }
- }
- }
-
/**
* clear related data if command of process instance is EXECUTE_TASK
* 1. find all task code from sub dag (only contains related task)
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
index d6d50ace80..eff0a3071f 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsService.java
@@ -27,6 +27,8 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nullable;
+
import lombok.NonNull;
public interface CuringParamsService {
@@ -80,6 +82,16 @@ public interface CuringParamsService {
@NonNull AbstractParameters parameters,
@NonNull ProcessInstance processInstance);
+ /**
+ * Parse workflow star parameter
+ */
+ Map<String, Property> parseWorkflowStartParam(@Nullable Map<String, String> cmdParam);
+
+ /**
+ * Parse workflow father parameter
+ */
+ Map<String, Property> parseWorkflowFatherParam(@Nullable Map<String, String> cmdParam);
+
/**
* preBuildBusinessParams
* @param processInstance
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
index e23c07b1b7..afcfae3fd6 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceImpl.java
@@ -28,6 +28,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETE
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_WORKFLOW_DEFINITION_NAME;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.PARAMETER_WORKFLOW_INSTANCE_ID;
+import org.apache.dolphinscheduler.common.constants.CommandKeyConstants;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.constants.DateConstants;
import org.apache.dolphinscheduler.common.enums.CommandType;
@@ -55,6 +56,8 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+
import lombok.NonNull;
import org.springframework.beans.factory.annotation.Autowired;
@@ -141,6 +144,28 @@ public class CuringParamsServiceImpl implements CuringParamsService {
return JSONUtils.toJsonString(globalParamList);
}
+ @Override
+ public Map<String, Property> parseWorkflowStartParam(@Nullable Map<String, String> cmdParam) {
+ if (cmdParam == null || !cmdParam.containsKey(CommandKeyConstants.CMD_PARAM_START_PARAMS)) {
+ return new HashMap<>();
+ }
+ String startParamJson = cmdParam.get(CommandKeyConstants.CMD_PARAM_START_PARAMS);
+ Map<String, String> startParamMap = JSONUtils.toMap(startParamJson);
+ return startParamMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey,
+ entry -> new Property(entry.getKey(), Direct.IN, DataType.VARCHAR, entry.getValue())));
+ }
+
+ @Override
+ public Map<String, Property> parseWorkflowFatherParam(@Nullable Map<String, String> cmdParam) {
+ if (cmdParam == null || !cmdParam.containsKey(CommandKeyConstants.CMD_PARAM_FATHER_PARAMS)) {
+ return new HashMap<>();
+ }
+ String startParamJson = cmdParam.get(CommandKeyConstants.CMD_PARAM_FATHER_PARAMS);
+ Map<String, String> startParamMap = JSONUtils.toMap(startParamJson);
+ return startParamMap.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey,
+ entry -> new Property(entry.getKey(), Direct.IN, DataType.VARCHAR, entry.getValue())));
+ }
+
/**
* the global parameters and local parameters used in the worker will be prepared here, and built-in parameters.
*
@@ -199,7 +224,7 @@ public class CuringParamsServiceImpl implements CuringParamsService {
}
if (MapUtils.isNotEmpty(cmdParam)) {
- prepareParamsMap.putAll(ParameterUtils.getUserDefParamsMap(cmdParam));
+ prepareParamsMap.putAll(parseWorkflowStartParam(cmdParam));
}
Iterator<Map.Entry<String, Property>> iter = prepareParamsMap.entrySet().iterator();
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index 091d6353fb..ba4def7e06 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -50,6 +50,7 @@ import org.apache.dolphinscheduler.service.exceptions.CronParseException;
import org.apache.dolphinscheduler.service.model.TaskNode;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;
@@ -194,4 +195,6 @@ public interface ProcessService {
void forceProcessInstanceSuccessByTaskInstanceId(Integer taskInstanceId);
void saveCommandTrigger(Integer commandId, Integer processInstanceId);
+
+ void setGlobalParamIfCommanded(ProcessDefinition processDefinition, Map<String, String> cmdParam);
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 93b257ba27..130a1c0993 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -28,8 +28,6 @@ import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.C
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
import static org.apache.dolphinscheduler.common.constants.CommandKeyConstants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID;
import static org.apache.dolphinscheduler.common.constants.Constants.LOCAL_PARAMS;
-import static org.apache.dolphinscheduler.plugin.task.api.enums.DataType.VARCHAR;
-import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
import static org.apache.dolphinscheduler.plugin.task.api.utils.DataQualityConstants.TASK_INSTANCE_ID;
import org.apache.dolphinscheduler.common.constants.CommandKeyConstants;
@@ -587,35 +585,32 @@ public class ProcessServiceImpl implements ProcessService {
return processInstance;
}
- private void setGlobalParamIfCommanded(ProcessDefinition processDefinition, Map<String, String> cmdParam) {
+ @Override
+ public void setGlobalParamIfCommanded(ProcessDefinition processDefinition, Map<String, String> cmdParam) {
+
// get start params from command param
- Map<String, String> startParamMap = new HashMap<>();
- if (cmdParam != null && cmdParam.containsKey(CommandKeyConstants.CMD_PARAM_START_PARAMS)) {
- String startParamJson = cmdParam.get(CommandKeyConstants.CMD_PARAM_START_PARAMS);
- startParamMap = JSONUtils.toMap(startParamJson);
- }
- Map<String, String> fatherParamMap = new HashMap<>();
- if (cmdParam != null && cmdParam.containsKey(CommandKeyConstants.CMD_PARAM_FATHER_PARAMS)) {
- String fatherParamJson = cmdParam.get(CommandKeyConstants.CMD_PARAM_FATHER_PARAMS);
- fatherParamMap = JSONUtils.toMap(fatherParamJson);
- }
- startParamMap.putAll(fatherParamMap);
+ Map<String, Property> fatherParam = curingGlobalParamsService.parseWorkflowFatherParam(cmdParam);
+ Map<String, Property> startParamMap = new HashMap<>(fatherParam);
+
+ Map<String, Property> currentStartParamMap = curingGlobalParamsService.parseWorkflowStartParam(cmdParam);
+ startParamMap.putAll(currentStartParamMap);
+
// set start param into global params
Map<String, String> globalMap = processDefinition.getGlobalParamMap();
List<Property> globalParamList = processDefinition.getGlobalParamList();
if (MapUtils.isNotEmpty(startParamMap) && globalMap != null) {
// start param to overwrite global param
for (Map.Entry<String, String> param : globalMap.entrySet()) {
- String val = startParamMap.get(param.getKey());
+ String val = startParamMap.get(param.getKey()).getValue();
if (val != null) {
param.setValue(val);
}
}
// start param to create new global param if global not exist
- for (Entry<String, String> startParam : startParamMap.entrySet()) {
+ for (Entry<String, Property> startParam : startParamMap.entrySet()) {
if (!globalMap.containsKey(startParam.getKey())) {
- globalMap.put(startParam.getKey(), startParam.getValue());
- globalParamList.add(new Property(startParam.getKey(), IN, VARCHAR, startParam.getValue()));
+ globalMap.put(startParam.getKey(), startParam.getValue().getValue());
+ globalParamList.add(startParam.getValue());
}
}
}
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
index 29b3ed58c1..96c9503249 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/expand/CuringParamsServiceTest.java
@@ -33,6 +33,8 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters;
+import org.apache.commons.collections4.MapUtils;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -234,4 +236,46 @@ public class CuringParamsServiceTest {
Assertions.assertEquals(propertyMap.get(TaskConstants.PARAMETER_WORKFLOW_DEFINITION_CODE).getValue(),
String.valueOf(processDefinition.getCode()));
}
+
+ @Test
+ public void testParseWorkflowStartParam() {
+ Map<String, Property> result = new HashMap<>();
+ // empty cmd param
+ Map<String, String> startParamMap = new HashMap<>();
+ result = dolphinSchedulerCuringGlobalParams.parseWorkflowStartParam(startParamMap);
+ Assertions.assertTrue(MapUtils.isEmpty(result));
+
+ // without key
+ startParamMap.put("testStartParam", "$[yyyyMMdd]");
+ result = dolphinSchedulerCuringGlobalParams.parseWorkflowStartParam(startParamMap);
+ Assertions.assertTrue(MapUtils.isEmpty(result));
+
+ startParamMap.put("StartParams", "{\"param1\":\"11111\", \"param2\":\"22222\"}");
+ result = dolphinSchedulerCuringGlobalParams.parseWorkflowStartParam(startParamMap);
+ Assertions.assertTrue(MapUtils.isNotEmpty(result));
+ Assertions.assertEquals(2, result.keySet().size());
+ Assertions.assertEquals("11111", result.get("param1").getValue());
+ Assertions.assertEquals("22222", result.get("param2").getValue());
+ }
+
+ @Test
+ public void testParseWorkflowFatherParam() {
+ Map<String, Property> result = new HashMap<>();
+ // empty cmd param
+ Map<String, String> startParamMap = new HashMap<>();
+ result = dolphinSchedulerCuringGlobalParams.parseWorkflowFatherParam(startParamMap);
+ Assertions.assertTrue(MapUtils.isEmpty(result));
+
+ // without key
+ startParamMap.put("testfatherParams", "$[yyyyMMdd]");
+ result = dolphinSchedulerCuringGlobalParams.parseWorkflowFatherParam(startParamMap);
+ Assertions.assertTrue(MapUtils.isEmpty(result));
+
+ startParamMap.put("fatherParams", "{\"param1\":\"11111\", \"param2\":\"22222\"}");
+ result = dolphinSchedulerCuringGlobalParams.parseWorkflowFatherParam(startParamMap);
+ Assertions.assertTrue(MapUtils.isNotEmpty(result));
+ Assertions.assertEquals(2, result.keySet().size());
+ Assertions.assertEquals("11111", result.get("param1").getValue());
+ Assertions.assertEquals("22222", result.get("param2").getValue());
+ }
}
(dolphinscheduler) 17/18: Set the workflow instance ready state to running in failover (#15572)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 9bb526855db74517c400417d50cf4e62d37e4c84
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Tue Feb 6 15:25:50 2024 +0800
Set the workflow instance ready state to running in failover (#15572)
(cherry picked from commit 1b32e28cc821daf940c264fecec23b2225446a67)
---
.../dolphinscheduler/service/process/ProcessServiceImpl.java | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index 1f27857513..fc906491d1 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -793,6 +793,14 @@ public class ProcessServiceImpl implements ProcessService {
break;
case RECOVER_TOLERANCE_FAULT_PROCESS:
// recover tolerance fault process
+ // If the workflow instance is in ready state, we will change to running, this can avoid the workflow
+ // instance
+ // status is not correct with taskInsatnce status
+ if (processInstance.getState() == WorkflowExecutionStatus.READY_PAUSE
+ || processInstance.getState() == WorkflowExecutionStatus.READY_STOP) {
+ // todo: If we handle the ready state in WorkflowExecuteRunnable then we can remove below code
+ processInstance.setState(WorkflowExecutionStatus.RUNNING_EXECUTION);
+ }
processInstance.setRecovery(Flag.YES);
processInstance.setRunTimes(runTime + 1);
runStatus = processInstance.getState();
(dolphinscheduler) 01/18: Exit JVM when OOM (#15538)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit be58ba69a9de8b5c3240ae0041e2e1e8184c4656
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Wed Jan 31 13:59:30 2024 +0800
Exit JVM when OOM (#15538)
(cherry picked from commit 25810a81dc528feedd81e44f8f57469286c0e8c7)
---
.../dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh | 1 +
dolphinscheduler-api/src/main/bin/jvm_args_env.sh | 1 +
dolphinscheduler-master/src/main/bin/jvm_args_env.sh | 1 +
dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh | 1 +
dolphinscheduler-worker/src/main/bin/jvm_args_env.sh | 1 +
5 files changed, 5 insertions(+)
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh
index d6ccc3c571..c668944139 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/bin/jvm_args_env.sh
@@ -24,6 +24,7 @@
-XX:+PrintGCDetails
-Xloggc:gc.log
+-XX:+ExitOnOutOfMemoryError
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof
diff --git a/dolphinscheduler-api/src/main/bin/jvm_args_env.sh b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh
index d6ccc3c571..c668944139 100644
--- a/dolphinscheduler-api/src/main/bin/jvm_args_env.sh
+++ b/dolphinscheduler-api/src/main/bin/jvm_args_env.sh
@@ -24,6 +24,7 @@
-XX:+PrintGCDetails
-Xloggc:gc.log
+-XX:+ExitOnOutOfMemoryError
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof
diff --git a/dolphinscheduler-master/src/main/bin/jvm_args_env.sh b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh
index c831da7ac7..c183f31bb1 100644
--- a/dolphinscheduler-master/src/main/bin/jvm_args_env.sh
+++ b/dolphinscheduler-master/src/main/bin/jvm_args_env.sh
@@ -24,6 +24,7 @@
-XX:+PrintGCDetails
-Xloggc:gc.log
+-XX:+ExitOnOutOfMemoryError
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof
diff --git a/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh b/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh
index 8bb0a623f9..42f8b1c656 100644
--- a/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh
+++ b/dolphinscheduler-standalone-server/src/main/bin/jvm_args_env.sh
@@ -24,6 +24,7 @@
-XX:+PrintGCDetails
-Xloggc:gc.log
+-XX:+ExitOnOutOfMemoryError
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof
diff --git a/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh b/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh
index 8bb0a623f9..42f8b1c656 100644
--- a/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh
+++ b/dolphinscheduler-worker/src/main/bin/jvm_args_env.sh
@@ -24,6 +24,7 @@
-XX:+PrintGCDetails
-Xloggc:gc.log
+-XX:+ExitOnOutOfMemoryError
-XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=dump.hprof
(dolphinscheduler) 06/18: fix: data quality can not use (#15551)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit acb1e0a09f00f15c3f504ae88428d4cddabbf2e2
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Mon Feb 5 09:54:06 2024 +0800
fix: data quality can not use (#15551)
* fix: data quality can not use
fix: #15468, #15249, #14858
---------
Co-authored-by: Rick Cheng <ri...@gmail.com>
Co-authored-by: Eric Gao <er...@gmail.com>
(cherry picked from commit 8efaa9fa1f1a5ab70dad0137df13da6480be59e9)
---
docs/docs/en/guide/data-quality.md | 10 +--
docs/docs/en/guide/resource/configuration.md | 5 +-
docs/docs/en/guide/upgrade/incompatible.md | 1 +
docs/docs/zh/guide/data-quality.md | 9 +--
docs/docs/zh/guide/resource/configuration.md | 5 +-
.../resources/docker/file-manage/common.properties | 5 +-
.../common/constants/DataSourceConstants.java | 2 +-
.../src/main/resources/common.properties | 5 +-
.../common/log/SensitiveDataConverterTest.java | 28 +++++++++
.../src/test/resources/common.properties | 5 +-
.../plugin/datasource/api/utils/CommonUtils.java | 61 ++++++++++++++++--
.../mysql/param/MySQLDataSourceProcessor.java | 14 +++++
.../resources/docker/file-manage/common.properties | 5 +-
.../master/runner/TaskExecutionContextFactory.java | 27 ++++----
.../spi/datasource/DefaultConnectionParam.java | 36 +++++++++++
.../plugin/task/api/model/JdbcInfo.java | 73 ++++------------------
.../plugin/task/api/utils/JdbcUrlParser.java | 24 ++++++-
.../plugin/task/api/utils/JdbcUrlParserTest.java | 18 ++++--
.../src/test/resources/common.properties | 5 +-
.../plugin/task/dq/DataQualityTask.java | 6 +-
20 files changed, 221 insertions(+), 123 deletions(-)
diff --git a/docs/docs/en/guide/data-quality.md b/docs/docs/en/guide/data-quality.md
index 2ed49b5f86..f6aa7a06b2 100644
--- a/docs/docs/en/guide/data-quality.md
+++ b/docs/docs/en/guide/data-quality.md
@@ -12,15 +12,7 @@ The execution logic of the data quality task is as follows:
- The current data quality task result is stored in the `t_ds_dq_execute_result` table of `dolphinscheduler`
`Worker` sends the task result to `Master`, after `Master` receives `TaskResponse`, it will judge whether the task type is `DataQualityTask`, if so, it will read the corresponding result from `t_ds_dq_execute_result` according to `taskInstanceId`, and then The result is judged according to the check mode, operator and threshold configured by the user.
- If the result is a failure, the corresponding operation, alarm or interruption will be performed according to the failure policy configured by the user.
-- Add config : `<server-name>/conf/common.properties`
-
-```properties
-# Change to specific version if you not use dev branch
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
-```
-
-- Please fill in `data-quality.jar.name` according to the actual package name.
-- If you package `data-quality` separately, remember to modify the package name to be consistent with `data-quality.jar.name`.
+- If you package `data-quality` separately, remember to modify the package name to be consistent with `data-quality.jar.name` in `common.properties` with attribute name `data-quality.jar.name`
- If the old version is upgraded and used, you need to execute the `sql` update script to initialize the database before running.
- `dolphinscheduler-data-quality-dev-SNAPSHOT.jar` was built with no dependencies. If a `JDBC` driver is required, you can set the `-jars` parameter in the `node settings` `Option Parameters`, e.g. `--jars /lib/jars/mysql-connector-java-8.0.16.jar`.
- Currently only `MySQL`, `PostgreSQL` and `HIVE` data sources have been tested, other data sources have not been tested yet.
diff --git a/docs/docs/en/guide/resource/configuration.md b/docs/docs/en/guide/resource/configuration.md
index 112b5458cc..42e1925e89 100644
--- a/docs/docs/en/guide/resource/configuration.md
+++ b/docs/docs/en/guide/resource/configuration.md
@@ -152,8 +152,9 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
+# if you re-build it alone, or auto discovery mechanism fail
+data-quality.jar.name=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md
index 0ecd94c8cb..4580a7d13d 100644
--- a/docs/docs/en/guide/upgrade/incompatible.md
+++ b/docs/docs/en/guide/upgrade/incompatible.md
@@ -10,6 +10,7 @@ This document records the incompatible updates between each version. You need to
* Remove the spark version of spark task ([#11860](https://github.com/apache/dolphinscheduler/pull/11860)).
* Change the default unix shell executor from sh to bash ([#12180](https://github.com/apache/dolphinscheduler/pull/12180)).
* Remove `deleteSource` in `download()` of `StorageOperate` ([#14084](https://github.com/apache/dolphinscheduler/pull/14084))
+* Remove default key for attribute `data-quality.jar.name` in `common.properties` ([#15551](https://github.com/apache/dolphinscheduler/pull/15551))
## 3.2.0
diff --git a/docs/docs/zh/guide/data-quality.md b/docs/docs/zh/guide/data-quality.md
index 95ca5e2d68..2a098a3216 100644
--- a/docs/docs/zh/guide/data-quality.md
+++ b/docs/docs/zh/guide/data-quality.md
@@ -13,14 +13,7 @@
>
## 注意事项
-添加配置信息：`<server-name>/conf/common.properties`
-
-```properties
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
-```
-
-- 这里的`data-quality.jar.name`请根据实际打包的名称来填写。
-- 如果单独打包`data-quality`的话，记得修改包名和`data-quality.jar.name`一致。
+- 如果单独打包`data-quality`的话，记得修改包名和`data-quality.jar.name`一致，配置内容在 `common.properties` 中的 `data-quality.jar.name`
- 如果是老版本升级使用，运行之前需要先执行`SQL`更新脚本进行数据库初始化。
- 当前 `dolphinscheduler-data-quality-dev-SNAPSHOT.jar` 是瘦包，不包含任何 `JDBC` 驱动。
如果有 `JDBC` 驱动需要，可以在`节点设置` `选项参数`处设置 `--jars` 参数，
diff --git a/docs/docs/zh/guide/resource/configuration.md b/docs/docs/zh/guide/resource/configuration.md
index c5c7f85620..57d0935e09 100644
--- a/docs/docs/zh/guide/resource/configuration.md
+++ b/docs/docs/zh/guide/resource/configuration.md
@@ -156,8 +156,9 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
+# if you re-build it alone, or auto discovery mechanism fail
+data-quality.jar.name=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties
index 000341f153..d43e55e822 100644
--- a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties
+++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties
@@ -84,8 +84,9 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
+# if you re-build it alone, or auto discovery mechanism fail
+data-quality.jar.name=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java
index 568eb8c6f5..11347942bd 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java
@@ -102,7 +102,7 @@ public class DataSourceConstants {
* dataSource sensitive param
*/
public static final String DATASOURCE_PASSWORD_REGEX =
- "(?<=((?i)password((\":\")|(\\\\\":\\\\\")|(=')))).*?(?=((\")|(\\\\\")|(')))";
+ "(?<=((?i)password((\" : \")|(\":\")|(\\\\\":\\\\\")|(=')))).*?(?=((\")|(\\\\\")|(')))";
/**
* datasource encryption salt
diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
index 28ebf4571d..451a0f734c 100644
--- a/dolphinscheduler-common/src/main/resources/common.properties
+++ b/dolphinscheduler-common/src/main/resources/common.properties
@@ -120,8 +120,9 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
+# if you re-build it alone, or auto discovery mechanism fail
+data-quality.jar.name=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
index c641c296b8..e6078ae95f 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
@@ -79,6 +79,34 @@ public class SensitiveDataConverterTest {
" }\n" +
"}");
+ // data quality
+ tcs.put("\"readers\" : [ {\n" +
+ " \"type\" : \"JDBC\",\n" +
+ " \"config\" : {\n" +
+ " \"database\" : \"dolphinscheduler\",\n" +
+ " \"password\" : \"view1\",\n" +
+ " \"driver\" : \"com.mysql.cj.jdbc.Driver\",\n" +
+ " \"user\" : \"root\",\n" +
+ " \"output_table\" : \"dolphinscheduler_users\",\n" +
+ " \"table\" : \"users\",\n" +
+ " \"url\" : \"jdbc:mysql://127.0.0.1:3307/dolphinscheduler?userSSL=true&enabledTLSProtocols=TLSv1.2\"\n"
+ +
+ " }\n" +
+ " } ]",
+ "\"readers\" : [ {\n" +
+ " \"type\" : \"JDBC\",\n" +
+ " \"config\" : {\n" +
+ " \"database\" : \"dolphinscheduler\",\n" +
+ " \"password\" : \"*****\",\n" +
+ " \"driver\" : \"com.mysql.cj.jdbc.Driver\",\n" +
+ " \"user\" : \"root\",\n" +
+ " \"output_table\" : \"dolphinscheduler_users\",\n" +
+ " \"table\" : \"users\",\n" +
+ " \"url\" : \"jdbc:mysql://127.0.0.1:3307/dolphinscheduler?userSSL=true&enabledTLSProtocols=TLSv1.2\"\n"
+ +
+ " }\n" +
+ " } ]");
+
for (String logMsg : tcs.keySet()) {
String maskedLog = SensitiveDataConverter.maskSensitiveData(logMsg);
logger.info("original parameter : {}", logMsg);
diff --git a/dolphinscheduler-common/src/test/resources/common.properties b/dolphinscheduler-common/src/test/resources/common.properties
index ef6cc3710e..107977df7f 100644
--- a/dolphinscheduler-common/src/test/resources/common.properties
+++ b/dolphinscheduler-common/src/test/resources/common.properties
@@ -115,8 +115,9 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
+# if you re-build it alone, or auto discovery mechanism fail
+data-quality.jar.name=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
index e6ecef287a..a4e64594c0 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java
@@ -36,17 +36,26 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import java.io.File;
import java.io.IOException;
+import java.util.Optional;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.core.io.ClassPathResource;
/**
* common utils
*/
+@Slf4j
public class CommonUtils {
private CommonUtils() {
throw new UnsupportedOperationException("Construct CommonUtils");
}
+ private static String DEFAULT_DATA_QUALITY_JAR_PATH = null;
+
private static final boolean IS_DEVELOP_MODE = PropertyUtils.getBoolean(Constants.DEVELOPMENT_STATE, true);
/**
@@ -123,14 +132,56 @@ public class CommonUtils {
return false;
}
- public static String getDataQualityJarName() {
- String dqsJarName = PropertyUtils.getString(DATA_QUALITY_JAR_NAME);
+ public static String getDataQualityJarPath() {
+ String dqsJarPath = PropertyUtils.getString(DATA_QUALITY_JAR_NAME);
- if (StringUtils.isEmpty(dqsJarName)) {
- return "dolphinscheduler-data-quality.jar";
+ if (StringUtils.isEmpty(dqsJarPath)) {
+ log.info("data quality jar path is empty, will try to get it from data quality jar name");
+ return getDefaultDataQualityJarPath();
}
- return dqsJarName;
+ return dqsJarPath;
+ }
+
+ private static String getDefaultDataQualityJarPath() {
+ if (StringUtils.isNotEmpty(DEFAULT_DATA_QUALITY_JAR_PATH)) {
+ return DEFAULT_DATA_QUALITY_JAR_PATH;
+ }
+ try {
+ // not standalone mode
+ String currentAbsolutePath = new ClassPathResource("./").getFile().getAbsolutePath();
+ String currentLibPath = currentAbsolutePath + "/../libs";
+ getDataQualityJarPathFromPath(currentLibPath).ifPresent(jarName -> DEFAULT_DATA_QUALITY_JAR_PATH = jarName);
+
+ // standalone mode
+ if (StringUtils.isEmpty(DEFAULT_DATA_QUALITY_JAR_PATH)) {
+ log.info(
+ "Can not get data quality jar from path {}, maybe service running in standalone mode, will try to find another path",
+ currentLibPath);
+ currentLibPath = currentAbsolutePath + "/../../worker-server/libs";
+ getDataQualityJarPathFromPath(currentLibPath)
+ .ifPresent(jarName -> DEFAULT_DATA_QUALITY_JAR_PATH = jarName);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("get default data quality jar path error", e);
+ }
+ log.info("get default data quality jar name: {}", DEFAULT_DATA_QUALITY_JAR_PATH);
+ return DEFAULT_DATA_QUALITY_JAR_PATH;
+ }
+
+ private static Optional<String> getDataQualityJarPathFromPath(String path) {
+ log.info("Try to get data quality jar from path {}", path);
+ File[] jars = new File(path).listFiles();
+ if (jars == null) {
+ log.warn("No data quality related jar found from path {}", path);
+ return Optional.empty();
+ }
+ for (File jar : jars) {
+ if (jar.getName().startsWith("dolphinscheduler-data-quality")) {
+ return Optional.of(jar.getAbsolutePath());
+ }
+ }
+ return Optional.empty();
}
/**
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java
index c1b91e5930..b954defdd1 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-mysql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/mysql/param/MySQLDataSourceProcessor.java
@@ -33,6 +33,7 @@ import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -115,6 +116,10 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
@Override
public String getJdbcUrl(ConnectionParam connectionParam) {
MySQLConnectionParam mysqlConnectionParam = (MySQLConnectionParam) connectionParam;
+ if (MapUtils.isNotEmpty(mysqlConnectionParam.getOther())) {
+ return String.format("%s?%s", mysqlConnectionParam.getJdbcUrl(),
+ transformOther(mysqlConnectionParam.getOther()));
+ }
return mysqlConnectionParam.getJdbcUrl();
}
@@ -182,4 +187,13 @@ public class MySQLDataSourceProcessor extends AbstractDataSourceProcessor {
&& !key.contains(ALLOW_URL_IN_LOCAL_IN_FILE_NAME);
}
+ private String transformOther(Map<String, String> otherMap) {
+ if (MapUtils.isNotEmpty(otherMap)) {
+ List<String> list = new ArrayList<>(otherMap.size());
+ otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
+ return String.join("&", list);
+ }
+ return null;
+ }
+
}
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
index f704bb60fc..b5f61011b3 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/common.properties
@@ -95,8 +95,9 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+# data quality option, it would auto discovery from libs directory. You can also specific the jar name in libs directory
+# if you re-build it alone, or auto discovery mechanism fail
+data-quality.jar.name=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java
index d436004719..ab1806ff67 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java
@@ -17,13 +17,6 @@
package org.apache.dolphinscheduler.server.master.runner;
-import static org.apache.dolphinscheduler.common.constants.Constants.ADDRESS;
-import static org.apache.dolphinscheduler.common.constants.Constants.DATABASE;
-import static org.apache.dolphinscheduler.common.constants.Constants.JDBC_URL;
-import static org.apache.dolphinscheduler.common.constants.Constants.OTHER;
-import static org.apache.dolphinscheduler.common.constants.Constants.PASSWORD;
-import static org.apache.dolphinscheduler.common.constants.Constants.SINGLE_SLASH;
-import static org.apache.dolphinscheduler.common.constants.Constants.USER;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAMESPACE_NAME;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.TASK_TYPE_DATA_QUALITY;
@@ -70,6 +63,8 @@ import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.master.exception.TaskExecutionContextCreateException;
import org.apache.dolphinscheduler.service.expand.CuringParamsService;
import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.DefaultConnectionParam;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections4.CollectionUtils;
@@ -80,7 +75,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.Properties;
import lombok.extern.slf4j.Slf4j;
@@ -400,15 +394,16 @@ public class TaskExecutionContextFactory {
dataSource.setUserName(hikariDataSource.getUsername());
JdbcInfo jdbcInfo = JdbcUrlParser.getJdbcInfo(hikariDataSource.getJdbcUrl());
if (jdbcInfo != null) {
- Properties properties = new Properties();
- properties.setProperty(USER, hikariDataSource.getUsername());
- properties.setProperty(PASSWORD, hikariDataSource.getPassword());
- properties.setProperty(DATABASE, jdbcInfo.getDatabase());
- properties.setProperty(ADDRESS, jdbcInfo.getAddress());
- properties.setProperty(OTHER, jdbcInfo.getParams());
- properties.setProperty(JDBC_URL, jdbcInfo.getAddress() + SINGLE_SLASH + jdbcInfo.getDatabase());
+ //
+ BaseConnectionParam baseConnectionParam = new DefaultConnectionParam();
+ baseConnectionParam.setUser(hikariDataSource.getUsername());
+ baseConnectionParam.setPassword(hikariDataSource.getPassword());
+ baseConnectionParam.setDatabase(jdbcInfo.getDatabase());
+ baseConnectionParam.setAddress(jdbcInfo.getAddress());
+ baseConnectionParam.setJdbcUrl(jdbcInfo.getJdbcUrl());
+ baseConnectionParam.setOther(jdbcInfo.getParams());
dataSource.setType(DbType.of(JdbcUrlParser.getDbType(jdbcInfo.getDriverName()).getCode()));
- dataSource.setConnectionParams(JSONUtils.toJsonString(properties));
+ dataSource.setConnectionParams(JSONUtils.toJsonString(baseConnectionParam));
}
return dataSource;
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DefaultConnectionParam.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DefaultConnectionParam.java
new file mode 100644
index 0000000000..a681ca6230
--- /dev/null
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/datasource/DefaultConnectionParam.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.spi.datasource;
+
+public class DefaultConnectionParam extends BaseConnectionParam {
+
+ @Override
+ public String toString() {
+ return "DefaultConnectionParam{"
+ + "user='" + user + '\''
+ + ", password='" + password + '\''
+ + ", address='" + address + '\''
+ + ", database='" + database + '\''
+ + ", jdbcUrl='" + jdbcUrl + '\''
+ + ", driverLocation='" + driverLocation + '\''
+ + ", driverClassName='" + driverClassName + '\''
+ + ", validationQuery='" + validationQuery + '\''
+ + ", other='" + other + '\''
+ + '}';
+ }
+}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/JdbcInfo.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/JdbcInfo.java
index 3e8f47ba7b..5e90dffbfe 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/JdbcInfo.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/JdbcInfo.java
@@ -17,9 +17,20 @@
package org.apache.dolphinscheduler.plugin.task.api.model;
+import java.util.Map;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
/**
* JdbcInfo
*/
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
public class JdbcInfo {
private String host;
@@ -30,67 +41,9 @@ public class JdbcInfo {
private String database;
- private String params;
+ private Map<String, String> params;
private String address;
- public String getHost() {
- return host;
- }
-
- public void setHost(String host) {
- this.host = host;
- }
-
- public String getPort() {
- return port;
- }
-
- public void setPort(String port) {
- this.port = port;
- }
-
- public String getDriverName() {
- return driverName;
- }
-
- public void setDriverName(String driverName) {
- this.driverName = driverName;
- }
-
- public String getDatabase() {
- return database;
- }
-
- public void setDatabase(String database) {
- this.database = database;
- }
-
- public String getParams() {
- return params;
- }
-
- public void setParams(String params) {
- this.params = params;
- }
-
- public String getAddress() {
- return address;
- }
-
- public void setAddress(String address) {
- this.address = address;
- }
-
- @Override
- public String toString() {
- return "JdbcInfo{"
- + "host='" + host + '\''
- + ", port='" + port + '\''
- + ", driverName='" + driverName + '\''
- + ", database='" + database + '\''
- + ", params='" + params + '\''
- + ", address='" + address + '\''
- + '}';
- }
+ private String jdbcUrl;
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParser.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParser.java
index 2ab0d69e23..e8e5ec299c 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParser.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParser.java
@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.api.utils;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.COLON;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.DOUBLE_SLASH;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EQUAL_SIGN;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.QUESTION;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.SEMICOLON;
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.SINGLE_SLASH;
@@ -30,6 +31,9 @@ import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.lang3.StringUtils;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* JdbcUrlParser
*/
@@ -105,8 +109,24 @@ public class JdbcUrlParser {
jdbcInfo.setHost(host);
jdbcInfo.setPort(port);
jdbcInfo.setDatabase(database);
- jdbcInfo.setParams(params);
- jdbcInfo.setAddress("jdbc:" + driverName + "://" + host + COLON + port);
+
+ if (StringUtils.isNotEmpty(params)) {
+ Map<String, String> others = new HashMap<>();
+ String[] paramList = params.split("&");
+ for (String param : paramList) {
+ // handle bad params
+ if (StringUtils.isEmpty(param) || !param.contains(EQUAL_SIGN)) {
+ continue;
+ }
+ String[] kv = param.split(EQUAL_SIGN);
+ others.put(kv[0], kv[1]);
+ }
+ jdbcInfo.setParams(others);
+ }
+
+ String address = "jdbc:" + driverName + "://" + host + COLON + port;
+ jdbcInfo.setAddress(address);
+ jdbcInfo.setJdbcUrl(address + SINGLE_SLASH + database);
return jdbcInfo;
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParserTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParserTest.java
index 9cef208187..bad9171bac 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParserTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/utils/JdbcUrlParserTest.java
@@ -34,10 +34,20 @@ public class JdbcUrlParserTest {
+ "useUnicode=true&characterEncoding=UTF-8");
if (jdbcInfo != null) {
String jdbcInfoStr = jdbcInfo.toString();
- String expected = "JdbcInfo{host='localhost', port='3306', "
- + "driverName='mysql', database='dolphinscheduler', "
- + "params='useUnicode=true&characterEncoding=UTF-8', "
- + "address='jdbc:mysql://localhost:3306'}";
+ String expected =
+ "JdbcInfo(host=localhost, port=3306, driverName=mysql, database=dolphinscheduler, " +
+ "params={useUnicode=true, characterEncoding=UTF-8}, address=jdbc:mysql://localhost:3306, jdbcUrl=jdbc:mysql://localhost:3306/dolphinscheduler)";
+ Assertions.assertEquals(expected, jdbcInfoStr);
+ }
+
+ // bad jdbc url case
+ jdbcInfo = JdbcUrlParser.getJdbcInfo("jdbc:mysql://localhost:3306/dolphinscheduler?"
+ + "useUnicode=true&&characterEncoding=UTF-8");
+ if (jdbcInfo != null) {
+ String jdbcInfoStr = jdbcInfo.toString();
+ String expected =
+ "JdbcInfo(host=localhost, port=3306, driverName=mysql, database=dolphinscheduler, " +
+ "params={useUnicode=true, characterEncoding=UTF-8}, address=jdbc:mysql://localhost:3306, jdbcUrl=jdbc:mysql://localhost:3306/dolphinscheduler)";
Assertions.assertEquals(expected, jdbcInfoStr);
}
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties
index 5fab54a143..9855d855e9 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/resources/common.properties
@@ -84,8 +84,9 @@ datasource.encryption.enable=false
# datasource encryption salt
datasource.encryption.salt=!@#$%^&*
-# data quality option
-data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+# data quality absolute path, it would auto discovery from libs directory. You can also specific the jar name in libs directory
+# if you re-build it alone, or auto discovery mechanism fail
+data-quality.jar.name=
#data-quality.error.output.path=/tmp/data-quality-error-data
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java
index 1bf1a6454c..ec8adc3eef 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-dataquality/src/main/java/org/apache/dolphinscheduler/plugin/task/dq/DataQualityTask.java
@@ -50,7 +50,6 @@ import org.apache.dolphinscheduler.plugin.task.dq.utils.SparkArgsUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
-import java.io.File;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -117,6 +116,7 @@ public class DataQualityTask extends AbstractYarnTask {
DataQualityConfiguration dataQualityConfiguration =
ruleManager.generateDataQualityParameter();
+ log.info("data quality configuration: {}", JSONUtils.toPrettyJsonString(dataQualityConfiguration));
dataQualityParameters
.getSparkParameters()
.setMainArgs("\""
@@ -177,9 +177,7 @@ public class DataQualityTask extends AbstractYarnTask {
protected void setMainJarName() {
ResourceInfo mainJar = new ResourceInfo();
- String basePath = System.getProperty("user.dir").replace(File.separator + "bin", "");
- mainJar.setResourceName(
- basePath + File.separator + "libs" + File.separator + CommonUtils.getDataQualityJarName());
+ mainJar.setResourceName(CommonUtils.getDataQualityJarPath());
dataQualityParameters.getSparkParameters().setMainJar(mainJar);
}
(dolphinscheduler) 05/18: Fix createFile with permission will not work (#15556)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit a76b4662beaa5d254f1e575df4d2e39d99e0be51
Author: Wenjun Ruan <we...@apache.org>
AuthorDate: Sat Feb 3 18:00:33 2024 +0800
Fix createFile with permission will not work (#15556)
(cherry picked from commit 4d15932ede770bb65786c697bf16cdd4f2cc02be)
---
.../dolphinscheduler/common/utils/FileUtils.java | 139 +++++----------------
.../common/utils/FileUtilsTest.java | 24 +++-
.../service/utils/ProcessUtils.java | 3 +-
.../plugin/storage/abs/AbsStorageOperator.java | 3 +-
.../plugin/storage/gcs/GcsStorageOperator.java | 3 +-
.../plugin/storage/obs/ObsStorageOperator.java | 3 +-
.../plugin/storage/oss/OssStorageOperator.java | 3 +-
.../plugin/storage/s3/S3StorageOperator.java | 3 +-
.../shell/BaseLinuxShellInterceptorBuilder.java | 8 +-
.../plugin/task/java/JavaTaskTest.java | 3 +-
.../worker/utils/TaskExecutionContextUtils.java | 3 +-
.../utils/TaskExecutionContextUtilsTest.java | 2 +-
12 files changed, 72 insertions(+), 125 deletions(-)
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index c71c5f2e47..1f3ccfed3d 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -25,8 +25,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VI
import static org.apache.dolphinscheduler.common.constants.Constants.UTF_8;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
-import org.apache.dolphinscheduler.common.exception.FileOperateException;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SystemUtils;
@@ -38,9 +36,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
-import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Set;
@@ -48,11 +44,10 @@ import java.util.zip.CRC32;
import java.util.zip.CheckedInputStream;
import lombok.NonNull;
+import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
-/**
- * file utils
- */
+@UtilityClass
@Slf4j
public class FileUtils {
@@ -62,14 +57,7 @@ public class FileUtils {
public static final String KUBE_CONFIG_FILE = "config";
- private static final String RWXR_XR_X = "rwxr-xr-x";
-
- private static final FileAttribute<Set<PosixFilePermission>> PERMISSION_755 =
- PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(RWXR_XR_X));
-
- private FileUtils() {
- throw new UnsupportedOperationException("Construct FileUtils");
- }
+ private static final Set<PosixFilePermission> PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x");
/**
* get download file absolute path and name
@@ -161,34 +149,6 @@ public class FileUtils {
return PropertyUtils.getString(RESOURCE_VIEW_SUFFIXES, RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE);
}
- /**
- * create directory if absent
- *
- * @param execLocalPath execute local path
- * @throws IOException errors
- */
- public static void createWorkDirIfAbsent(String execLocalPath) throws IOException {
- // if work dir exists, first delete
- File execLocalPathFile = new File(execLocalPath);
-
- if (execLocalPathFile.exists()) {
- try {
- org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
- } catch (Exception ex) {
- if (ex instanceof NoSuchFileException || ex.getCause() instanceof NoSuchFileException) {
- // this file is already be deleted.
- } else {
- throw ex;
- }
- }
- }
-
- // create work dir
- org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
- String mkdirLog = "create dir success " + execLocalPath;
- log.info(mkdirLog);
- }
-
/**
* write content to file ,if parent path not exists, it will do one's utmost to mkdir
*
@@ -231,25 +191,6 @@ public class FileUtils {
org.apache.commons.io.FileUtils.deleteQuietly(new File(filename));
}
- /**
- * Gets all the parent subdirectories of the parentDir directory
- *
- * @param parentDir parent dir
- * @return all dirs
- */
- public static File[] getAllDir(String parentDir) {
- if (parentDir == null || "".equals(parentDir)) {
- throw new RuntimeException("parentDir can not be empty");
- }
-
- File file = new File(parentDir);
- if (!file.exists() || !file.isDirectory()) {
- throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
- }
-
- return file.listFiles(File::isDirectory);
- }
-
/**
* Get Content
*
@@ -325,59 +266,47 @@ public class FileUtils {
return crcString;
}
- public static void setFileOwner(Path filePath, String fileOwner) throws FileOperateException {
- try {
- // We use linux command to set the file owner, since jdk api will not use sudo.
- String command = String.format("sudo chown %s %s", fileOwner, filePath.toString());
- Runtime.getRuntime().exec(command);
- Process process = Runtime.getRuntime().exec(command);
- int exitCode = process.waitFor();
- if (0 != exitCode) {
- throw new FileOperateException(
- "Set file: " + filePath + " to owner: " + fileOwner + " failed, existCode(" + exitCode + ")");
- }
- } catch (FileOperateException ex) {
- throw ex;
- } catch (Exception ex) {
- throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
-
+ public static void createFileWith755(@NonNull Path path) throws IOException {
+ if (SystemUtils.IS_OS_WINDOWS) {
+ Files.createFile(path);
+ } else {
+ Files.createFile(path);
+ Files.setPosixFilePermissions(path, PERMISSION_755);
}
}
- public static void setDirectoryOwner(Path filePath, String fileOwner) throws FileOperateException {
- try {
- // We use linux command to set the file owner, since jdk api will not use sudo.
- String command = String.format("sudo chown -R %s %s", fileOwner, filePath.toString());
- Runtime.getRuntime().exec(command);
- Process process = Runtime.getRuntime().exec(command);
- int exitCode = process.waitFor();
- if (0 != exitCode) {
- throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner
- + " failed, existCode(" + exitCode + ")");
+ public static void createDirectoryWith755(@NonNull Path path) throws IOException {
+ if (path.toFile().exists()) {
+ return;
+ }
+ if (OSUtils.isWindows()) {
+ Files.createDirectories(path);
+ } else {
+ Path parent = path.getParent();
+ if (parent != null && !parent.toFile().exists()) {
+ createDirectoryWith755(parent);
}
- } catch (FileOperateException ex) {
- throw ex;
- } catch (Exception ex) {
- throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
+
+ Files.createDirectory(path);
+ Files.setPosixFilePermissions(path, PERMISSION_755);
}
}
- public static void createDirectoryIfNotPresent(Path path) throws IOException {
- if (Files.exists(path)) {
+ public static void setFileTo755(File file) throws IOException {
+ if (OSUtils.isWindows()) {
return;
}
- Files.createDirectories(path);
- }
-
- /**
- * Create a file with '755'.
- */
- public static void createFileWith755(@NonNull Path path) throws IOException {
- if (SystemUtils.IS_OS_WINDOWS) {
- Files.createFile(path);
- } else {
- Files.createFile(path, PERMISSION_755);
+ if (file.isFile()) {
+ Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
+ return;
+ }
+ Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
+ File[] files = file.listFiles();
+ if (files != null) {
+ for (File f : files) {
+ setFileTo755(f);
+ }
}
}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
index 0ce33bef52..f06ece2a56 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
@@ -19,8 +19,13 @@ package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
+import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -55,12 +60,23 @@ public class FileUtilsTest {
}
@Test
- public void testCreateWorkDirIfAbsent() {
+ public void createDirectoryWith755() throws IOException {
+ Path path = Paths.get("/tmp/createWorkDirAndUserIfAbsent");
try {
- FileUtils.createWorkDirIfAbsent("/tmp/createWorkDirAndUserIfAbsent");
- Assertions.assertTrue(true);
+ FileUtils.createDirectoryWith755(path);
+ File file = path.toFile();
+ Assertions.assertTrue(file.exists());
+ Assertions.assertTrue(file.isDirectory());
+ Assertions.assertTrue(file.canExecute());
+ Assertions.assertTrue(file.canRead());
+ Assertions.assertTrue(file.canWrite());
+
+ FileUtils.createDirectoryWith755(Paths.get("/"));
} catch (Exception e) {
- Assertions.fail();
+ e.printStackTrace();
+ Assertions.fail(e.getMessage());
+ } finally {
+ Files.deleteIfExists(path);
}
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java
index 6b97590c89..22d17d0fd3 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java
@@ -30,6 +30,7 @@ import org.apache.commons.lang3.SystemUtils;
import java.io.File;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -176,7 +177,7 @@ public class ProcessUtils {
taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTaskInstanceId()));
}
- FileUtils.createWorkDirIfAbsent(taskExecutionContext.getExecutePath());
+ FileUtils.createDirectoryWith755(Paths.get(taskExecutionContext.getExecutePath()));
org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils.cancelApplication(taskExecutionContext);
return appIds;
} else {
diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java
index bb899030c5..d470545bab 100644
--- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java
+++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java
@@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -170,7 +171,7 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
- Files.createDirectories(dstFile.getParentFile().toPath());
+ FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
BlobClient blobClient = blobContainerClient.getBlobClient(srcFilePath);
diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
index 9d09c2c9bb..e4176dc58e 100644
--- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
+++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
@@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -143,7 +144,7 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
- Files.createDirectories(dstFile.getParentFile().toPath());
+ FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
Blob blob = gcsStorage.get(BlobId.of(bucketName, srcFilePath));
diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java
index b660c59ab4..b644210048 100644
--- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java
+++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java
@@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -187,7 +188,7 @@ public class ObsStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
- Files.createDirectories(dstFile.getParentFile().toPath());
+ FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
ObsObject obsObject = obsClient.getObject(bucketName, srcFilePath);
try (
diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
index 2b2d18fd27..61754b52a7 100644
--- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
+++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
@@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.factory.OssClientFactory;
import org.apache.dolphinscheduler.common.model.OssConnection;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -213,7 +214,7 @@ public class OssStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
- Files.createDirectories(dstFile.getParentFile().toPath());
+ FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
OSSObject ossObject = ossClient.getObject(bucketName, srcFilePath);
try (
diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
index 884c672d53..a13611316e 100644
--- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
+++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
@@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@@ -200,7 +201,7 @@ public class S3StorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
- Files.createDirectories(dstFile.getParentFile().toPath());
+ FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
S3Object o = s3Client.getObject(bucketName, srcFilePath);
try (
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java
index c391ca87a7..afc2edc586 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java
@@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.plugin.task.api.shell;
-import org.apache.dolphinscheduler.common.constants.TenantConstants;
-import org.apache.dolphinscheduler.common.exception.FileOperateException;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.AbstractCommandExecutorConstants;
@@ -71,12 +69,8 @@ public abstract class BaseLinuxShellInterceptorBuilder<T extends BaseLinuxShellI
"****************************** Script Content *****************************************************************");
}
- protected List<String> generateBootstrapCommand() throws FileOperateException {
+ protected List<String> generateBootstrapCommand() {
if (sudoEnable) {
- if (!TenantConstants.BOOTSTRAPT_SYSTEM_USER.equals(runUser)) {
- // Set the tenant owner as the working directory
- FileUtils.setDirectoryOwner(Paths.get(shellDirectory), runUser);
- }
return bootstrapCommandInSudoMode();
}
return bootstrapCommandInNormalMode();
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java
index 7ea5a792b7..55756241ce 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java
@@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR;
import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAVA;
+import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@@ -155,7 +156,7 @@ public class JavaTaskTest {
try {
Path path = Paths.get(fileName);
if (!Files.exists(path)) {
- Files.createDirectories(path);
+ FileUtils.createDirectoryWith755(path);
}
javaTask.createJavaSourceFileIfNotExists(sourceCode, fileName);
} finally {
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
index ddd489adeb..3cda6ac099 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
@@ -97,7 +97,7 @@ public class TaskExecutionContextUtils {
log.warn("The TaskInstance WorkingDirectory: {} is exist, will recreate again",
taskInstanceWorkingDirectory);
}
- Files.createDirectories(Paths.get(taskInstanceWorkingDirectory));
+ FileUtils.createDirectoryWith755(Paths.get(taskInstanceWorkingDirectory));
taskExecutionContext.setExecutePath(taskInstanceWorkingDirectory);
taskExecutionContext.setAppInfoPath(FileUtils.getAppInfoPath(taskInstanceWorkingDirectory));
@@ -137,6 +137,7 @@ public class TaskExecutionContextUtils {
storageOperate.download(resourceAbsolutePathInStorage, resourceAbsolutePathInLocal, true);
log.debug("Download resource file {} under: {} successfully", resourceAbsolutePathInStorage,
resourceAbsolutePathInLocal);
+ FileUtils.setFileTo755(file);
WorkerServerMetrics
.recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime);
WorkerServerMetrics
diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
index b73d073ef6..ff1eb26e12 100644
--- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
+++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
@@ -49,7 +49,7 @@ class TaskExecutionContextUtilsTest {
try {
// Test if the working directory is exist
// will delete it and recreate
- Files.createDirectories(Paths.get(taskWorkingDirectory));
+ FileUtils.createDirectoryWith755(Paths.get(taskWorkingDirectory));
Files.createFile(Paths.get(taskWorkingDirectory, "text.txt"));
Assertions.assertTrue(Files.exists(Paths.get(taskWorkingDirectory, "text.txt")));
(dolphinscheduler) 09/18: fix: ddl without drop exists (#14128)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit d87a1acd19bfb8ff5c4f3bacfbf71c84d720aa36
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Mon Feb 5 12:20:07 2024 +0800
fix: ddl without drop exists (#14128)
Co-authored-by: Rick Cheng <ri...@gmail.com>
(cherry picked from commit d8e820c4b9fa70e6d7c8368a8dc17082f8b15db0)
---
.../src/main/resources/sql/dolphinscheduler_mysql.sql | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
index 626f6f8b46..9509b1b99f 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
@@ -2092,6 +2092,10 @@ CREATE TABLE `t_ds_fav_task`
AUTO_INCREMENT = 1
DEFAULT CHARSET = utf8 COLLATE = utf8_bin;
+-- ----------------------------
+-- Table structure for t_ds_trigger_relation
+-- ----------------------------
+DROP TABLE IF EXISTS `t_ds_trigger_relation`;
CREATE TABLE `t_ds_trigger_relation` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`trigger_type` int(11) NOT NULL DEFAULT '0' COMMENT '0 process 1 task',