You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ca...@apache.org on 2022/04/19 07:24:03 UTC
[dolphinscheduler] branch dev updated: [Bug] [API-9558]fix homepage task instance count method to use submit time to recount (#9559)
This is an automated email from the ASF dual-hosted git repository.
caishunfeng pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new c5b7e5adff [Bug] [API-9558]fix homepage task instance count method to use submit time to recount (#9559)
c5b7e5adff is described below
commit c5b7e5adff40988f3dfd4f75dddb0b597509fff4
Author: Tq <ti...@gmail.com>
AuthorDate: Tue Apr 19 15:23:57 2022 +0800
[Bug] [API-9558]fix homepage task instance count method to use submit time to recount (#9559)
* fix homepage task instance count method to use submit time to recount
* fix homepage task instance count method to use submit time to recount
* fix homepage task instance count method to use submit time to recount
* fix homepage task instance count method JUNIT
* fix homepage task instance count method JUNIT
* fix homepage task instance count method JUNIT
---
.../api/service/DataAnalysisService.java | 33 +++++++++++----
.../api/service/impl/DataAnalysisServiceImpl.java | 49 +++++++++++++++++-----
.../dao/mapper/TaskInstanceMapper.java | 25 ++++++++---
.../dao/mapper/TaskInstanceMapper.xml | 27 ++++++++++++
.../plugin/task/api/enums/ExecutionStatus.java | 3 +-
5 files changed, 111 insertions(+), 26 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java
index 80810ed40e..cb4639acfc 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataAnalysisService.java
@@ -17,8 +17,12 @@
package org.apache.dolphinscheduler.api.service;
+import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.ibatis.annotations.Param;
+import java.util.Date;
+import java.util.List;
import java.util.Map;
/**
@@ -29,10 +33,10 @@ public interface DataAnalysisService {
/**
* statistical task instance status data
*
- * @param loginUser login user
+ * @param loginUser login user
* @param projectCode project code
- * @param startDate start date
- * @param endDate end date
+ * @param startDate start date
+ * @param endDate end date
* @return task state count data
*/
Map<String, Object> countTaskStateByProject(User loginUser, long projectCode, String startDate, String endDate);
@@ -40,20 +44,20 @@ public interface DataAnalysisService {
/**
* statistical process instance status data
*
- * @param loginUser login user
+ * @param loginUser login user
* @param projectCode project code
- * @param startDate start date
- * @param endDate end date
+ * @param startDate start date
+ * @param endDate end date
* @return process instance state count data
*/
Map<String, Object> countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate);
/**
* statistics the process definition quantities of a certain person
- *
+ * <p>
* We only need projects which users have permission to see to determine whether the definition belongs to the user or not.
*
- * @param loginUser login user
+ * @param loginUser login user
* @param projectCode project code
* @return definition count data
*/
@@ -75,4 +79,17 @@ public interface DataAnalysisService {
*/
Map<String, Object> countQueueState(User loginUser);
+ /**
+ * Statistics task instance group by given project codes list
+ * <p>
+ * We only need project codes to determine whether the task instance belongs to the user or not.
+ *
+ * @param startTime Statistics start time
+ * @param endTime Statistics end time
+ * @param projectCodes Project codes list to filter
+ * @return List of ExecuteStatusCount
+ */
+ List<ExecuteStatusCount> countTaskInstanceAllStatesByProjectCodes(@Param("startTime") Date startTime,
+ @Param("endTime") Date endTime,
+ @Param("projectCodes") Long[] projectCodes);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
index 6617207a99..88449a49f0 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataAnalysisServiceImpl.java
@@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.dto.CommandStateCount;
import org.apache.dolphinscheduler.api.dto.DefineUserDto;
import org.apache.dolphinscheduler.api.dto.TaskCountDto;
+import org.apache.dolphinscheduler.api.dto.TaskStateCount;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.DataAnalysisService;
import org.apache.dolphinscheduler.api.service.ProjectService;
@@ -52,6 +53,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -101,11 +103,11 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
public Map<String, Object> countTaskStateByProject(User loginUser, long projectCode, String startDate, String endDate) {
return countStateByProject(
- loginUser,
- projectCode,
- startDate,
- endDate,
- (start, end, projectCodes) -> this.taskInstanceMapper.countTaskInstanceStateByProjectCodes(start, end, projectCodes));
+ loginUser,
+ projectCode,
+ startDate,
+ endDate,
+ this::countTaskInstanceAllStatesByProjectCodes);
}
/**
@@ -119,15 +121,15 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
*/
@Override
public Map<String, Object> countProcessInstanceStateByProject(User loginUser, long projectCode, String startDate, String endDate) {
- Map<String, Object> result = this.countStateByProject(
+ Map<String, Object> result = this.countStateByProject(
loginUser,
projectCode,
startDate,
endDate,
- (start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByProjectCodes(start, end, projectCodes));
+ (start, end, projectCodes) -> this.processInstanceMapper.countInstanceStateByProjectCodes(start, end, projectCodes));
// process state count needs to remove state of forced success
if (result.containsKey(Constants.STATUS) && result.get(Constants.STATUS).equals(Status.SUCCESS)) {
- ((TaskCountDto)result.get(Constants.DATA_LIST)).removeStateFromCountList(ExecutionStatus.FORCED_SUCCESS);
+ ((TaskCountDto) result.get(Constants.DATA_LIST)).removeStateFromCountList(ExecutionStatus.FORCED_SUCCESS);
}
return result;
}
@@ -163,9 +165,9 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
}
}
- List<ExecuteStatusCount> processInstanceStateCounts = new ArrayList<>();
Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser)
- : new Long[] {projectCode};
+ : new Long[]{projectCode};
+ List<ExecuteStatusCount> processInstanceStateCounts = new ArrayList<>();
if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) {
processInstanceStateCounts = instanceStateCounter.apply(start, end, projectCodeArray);
@@ -203,7 +205,7 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
List<DefinitionGroupByUser> defineGroupByUsers = new ArrayList<>();
Long[] projectCodeArray = projectCode == 0 ? getProjectCodesArrays(loginUser)
- : new Long[] {projectCode};
+ : new Long[]{projectCode};
if (projectCodeArray.length != 0 || loginUser.getUserType() == UserType.ADMIN_USER) {
defineGroupByUsers = processDefinitionMapper.countDefinitionByProjectCodes(projectCodeArray);
}
@@ -288,4 +290,29 @@ public class DataAnalysisServiceImpl extends BaseServiceImpl implements DataAnal
return result;
}
+ @Override
+ public List<ExecuteStatusCount> countTaskInstanceAllStatesByProjectCodes(Date startTime, Date endTime, Long[] projectCodes) {
+ Optional<List<ExecuteStatusCount>> startTimeStates = Optional.ofNullable(this.taskInstanceMapper.countTaskInstanceStateByProjectCodes(startTime, endTime, projectCodes));
+
+ List<ExecutionStatus> allState = Arrays.stream(ExecutionStatus.values()).collect(Collectors.toList());
+ List<ExecutionStatus> needRecountState;
+ if (startTimeStates.isPresent() && startTimeStates.get().size() != 0) {
+ List<ExecutionStatus> instanceState = startTimeStates.get().stream().map(ExecuteStatusCount::getExecutionStatus).collect(Collectors.toList());
+ //value 0 state need to recount by submit time
+ needRecountState = allState.stream().filter(ele -> !instanceState.contains(ele)).collect(Collectors.toList());
+ if (needRecountState.size() == 0) {
+ return startTimeStates.get();
+ }
+ } else {
+ needRecountState = allState;
+ }
+
+ //use submit time to recount when 0
+ //if have any issues with this code, should change to specified states 0 8 9 17 not state count is 0
+ List<ExecuteStatusCount> recounts = this.taskInstanceMapper
+ .countTaskInstanceStateByProjectCodesAndStatesBySubmitTime(startTime, endTime, projectCodes, needRecountState);
+ startTimeStates.orElseGet(ArrayList::new).addAll(recounts);
+
+ return startTimeStates.orElse(null);
+ }
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java
index a79a07ca27..f98996952d 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.java
@@ -17,19 +17,17 @@
package org.apache.dolphinscheduler.dao.mapper;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.plugin.task.api.enums.ExecutionStatus;
-
import org.apache.ibatis.annotations.Param;
import java.util.Date;
import java.util.List;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-
/**
* task instance mapper interface
*/
@@ -58,7 +56,7 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
@Param("taskIds") int[] taskIds);
/**
- * Statistics task instance group by given project codes list
+ * Statistics task instance group by given project codes list by start time
* <p>
* We only need project codes to determine whether the task instance belongs to the user or not.
*
@@ -71,6 +69,21 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
@Param("endTime") Date endTime,
@Param("projectCodes") Long[] projectCodes);
+ /**
+ * Statistics task instance group by given project codes list by submit time
+ * <p>
+ * We only need project codes to determine whether the task instance belongs to the user or not.
+ *
+ * @param startTime Statistics start time
+ * @param endTime Statistics end time
+ * @param projectCodes Project codes list to filter
+ * @return List of ExecuteStatusCount
+ */
+ List<ExecuteStatusCount> countTaskInstanceStateByProjectCodesAndStatesBySubmitTime(@Param("startTime") Date startTime,
+ @Param("endTime") Date endTime,
+ @Param("projectCodes") Long[] projectCodes,
+ @Param("states") List<ExecutionStatus> states);
+
IPage<TaskInstance> queryTaskInstanceListPaging(IPage<TaskInstance> page,
@Param("projectCode") Long projectCode,
@Param("processInstanceId") Integer processInstanceId,
@@ -84,5 +97,5 @@ public interface TaskInstanceMapper extends BaseMapper<TaskInstance> {
@Param("endTime") Date endTime
);
- List<TaskInstance> loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId,@Param("status") int status);
+ List<TaskInstance> loadAllInfosNoRelease(@Param("processInstanceId") int processInstanceId, @Param("status") int status);
}
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
index 8ea8459cd4..adc470c197 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
@@ -89,6 +89,33 @@
</if>
group by t.state
</select>
+ <select id="countTaskInstanceStateByProjectCodesAndStatesBySubmitTime" resultType="org.apache.dolphinscheduler.dao.entity.ExecuteStatusCount">
+ select
+ state, count(0) as count
+ from t_ds_task_instance t
+ left join t_ds_task_definition_log d on d.code=t.task_code and d.version=t.task_definition_version
+ left join t_ds_project p on p.code=d.project_code
+ where 1=1
+ <if test="states != null and states.size != 0">
+ and t.state in
+ <foreach collection="states" index="index" item="state" open="(" separator="," close=")">
+ #{state.code}
+ </foreach>
+ </if>
+ <if test="projectCodes != null and projectCodes.length != 0">
+ and d.project_code in
+ <foreach collection="projectCodes" index="index" item="i" open="(" separator="," close=")">
+ #{i}
+ </foreach>
+ </if>
+ <if test="startTime != null">
+ and t.submit_time <![CDATA[ > ]]> #{startTime}
+ </if>
+ <if test="endTime != null">
+ and t.submit_time <![CDATA[ <= ]]> #{endTime}
+ </if>
+ group by t.state
+ </select>
<select id="queryByInstanceIdAndName" resultType="org.apache.dolphinscheduler.dao.entity.TaskInstance">
select
<include refid="baseSql"/>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java
index 8d2774c531..82c8db12d4 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/enums/ExecutionStatus.java
@@ -45,6 +45,7 @@ public enum ExecutionStatus {
* 14 serial wait
* 15 ready block
* 16 block
+ * 17 dispatch
*/
SUBMITTED_SUCCESS(0, "submit success"),
RUNNING_EXECUTION(1, "running"),
@@ -108,7 +109,7 @@ public enum ExecutionStatus {
*/
public boolean typeIsFinished() {
return typeIsSuccess() || typeIsFailure() || typeIsCancel() || typeIsPause()
- || typeIsStop() || typeIsBlock();
+ || typeIsStop() || typeIsBlock();
}
/**