You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by al...@apache.org on 2022/07/22 12:32:00 UTC
[incubator-linkis] 02/06: undonetask interface performance optimization close #2504
This is an automated email from the ASF dual-hosted git repository.
alexkun pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
commit 2f9e9bd9b9309a154dbf04d124e5c9a156f12d76
Author: peacewong <wp...@gmail.com>
AuthorDate: Fri Jul 22 16:30:49 2022 +0800
undonetask interface performance optimization close #2504
---
.../impl/DefaultLabelManagerPersistence.java | 2 +-
.../linkis/jobhistory/dao/JobHistoryMapper.java | 38 +++++++++++--
.../jobhistory/dao/impl/JobHistoryMapper.xml | 40 ++++++++++++++
.../jobhistory/restful/api/QueryRestfulApi.java | 49 ++++++++++++++++-
.../jobhistory/service/JobHistoryQueryService.java | 4 +-
.../service/impl/JobHistoryQueryServiceImpl.scala | 64 +++++++++++++++++++---
6 files changed, 180 insertions(+), 17 deletions(-)
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
index 2e4798099..da46fa111 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
+++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
@@ -139,7 +139,7 @@ public class DefaultLabelManagerPersistence implements LabelManagerPersistence {
@Override
public PersistenceLabel getLabel(int id) {
- PersistenceLabel persistenceLabel = labelManagerMapper.getLabel(id);
+ PersistenceLabel persistenceLabel = labelManagerMapper.getLabel(id);
PersistenceUtils.setValue(persistenceLabel);
return labelManagerMapper.getLabel(id);
}
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java
index 12214723e..3c72f6e96 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java
@@ -32,22 +32,22 @@ public interface JobHistoryMapper {
void updateJobHistory(JobHistory jobReq);
- List<JobHistory> search(
+ List<JobHistory> searchWithIdOrderAsc(
@Param("id") Long id,
@Param("umUser") String username,
@Param("status") List<String> status,
@Param("startDate") Date startDate,
@Param("endDate") Date endDate,
- @Param("engineType") String engineType,
- @Param("startId") Long startId);
+ @Param("engineType") String engineType);
- List<JobHistory> searchWithIdOrderAsc(
+ List<JobHistory> search(
@Param("id") Long id,
@Param("umUser") String username,
@Param("status") List<String> status,
@Param("startDate") Date startDate,
@Param("endDate") Date endDate,
- @Param("engineType") String engineType);
+ @Param("engineType") String engineType,
+ @Param("startId") Long startId);
List<JobHistory> searchWithUserCreator(
@Param("id") Long id,
@@ -71,5 +71,33 @@ public interface JobHistoryMapper {
@Param("engineType") String engineType,
@Param("startId") Long startId);
+ Integer countUndoneTaskNoCreator(
+ @Param("umUser") String username,
+ @Param("status") List<String> status,
+ @Param("startDate") Date startDate,
+ @Param("endDate") Date endDate,
+ @Param("engineType") String engineType,
+ @Param("startId") Long startId);
+
+ Integer countUndoneTaskWithUserCreator(
+ @Param("umUser") String username,
+ @Param("userCreatorKey") String userCreatorKey,
+ @Param("userCreatorValue") String userCreator,
+ @Param("status") List<String> status,
+ @Param("startDate") Date startDate,
+ @Param("endDate") Date endDate,
+ @Param("engineType") String engineType,
+ @Param("startId") Long startId);
+
+ Integer countUndoneTaskWithCreatorOnly(
+ @Param("umUser") String username,
+ @Param("userCreatorKey") String userCreatorKey,
+ @Param("creator") String userCreator,
+ @Param("status") List<String> status,
+ @Param("startDate") Date startDate,
+ @Param("endDate") Date endDate,
+ @Param("engineType") String engineType,
+ @Param("startId") Long startId);
+
String selectJobHistoryStatusForUpdate(Long jobId);
}
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml
index 2a2823c1f..9dc88cd2e 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml
@@ -173,4 +173,44 @@
WHERE
id = #{jobId} FOR UPDATE
</select>
+
+ <select id="countUndoneTaskNoCreator" useCache="true" resultType="java.lang.Integer" >
+ /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history
+ <where>
+ <if test="startId != null"> id >= #{startId}</if>
+ <if test="umUser != null">and submit_user = #{umUser}</if>
+ <if test="engineType != null">and engine_type = #{engineType}</if>
+ <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if>
+ <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if>
+ </where>
+ </select>
+
+ <select id="countUndoneTaskWithUserCreator" useCache="true" resultType="java.lang.Integer" >
+ /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history
+ <where>
+ <if test="startId != null"> id >= #{startId}</if>
+ <if test="umUser != null">and submit_user = #{umUser}</if>
+ <if test="engineType != null">and engine_type = #{engineType}</if>
+ <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if>
+ <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if>
+ <if test="userCreatorKey != null and userCreatorValue != null">
+ and LOCATE('"${userCreatorKey}":"${userCreatorValue}', labels) > 0
+ </if>
+ </where>
+ </select>
+
+ <select id="countUndoneTaskWithCreatorOnly" useCache="true" resultType="java.lang.Integer" >
+ /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history
+ <where>
+ <if test="startId != null"> id >= #{startId}</if>
+ <if test="umUser != null">and submit_user = #{umUser}</if>
+ <if test="engineType != null">and engine_type = #{engineType}</if>
+ <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if>
+ <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if>
+ <if test="userCreatorKey != null and creator != null">
+ and labels like '%"${userCreatorKey}":"%-${creator}%'
+ </if>
+ </where>
+ </select>
+
</mapper>
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java
index 117b92bb1..89bb49f26 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java
@@ -209,8 +209,8 @@ public class QueryRestfulApi {
}
/** Method list should not contain subjob, which may cause performance problems. */
- @RequestMapping(path = "/listundone", method = RequestMethod.GET)
- public Message listundone(
+ @RequestMapping(path = "/listundonetasks", method = RequestMethod.GET)
+ public Message listundonetasks(
HttpServletRequest req,
@RequestParam(value = "startDate", required = false) Long startDate,
@RequestParam(value = "endDate", required = false) Long endDate,
@@ -281,4 +281,49 @@ public class QueryRestfulApi {
.data(TaskConstant.TASKS, vos)
.data(JobRequestConstants.TOTAL_PAGE(), total);
}
+
+ /** Method list should not contain subjob, which may cause performance problems. */
+ @RequestMapping(path = "/listundone", method = RequestMethod.GET)
+ public Message listundone(
+ HttpServletRequest req,
+ @RequestParam(value = "startDate", required = false) Long startDate,
+ @RequestParam(value = "endDate", required = false) Long endDate,
+ @RequestParam(value = "pageNow", required = false) Integer pageNow,
+ @RequestParam(value = "pageSize", required = false) Integer pageSize,
+ @RequestParam(value = "startTaskID", required = false) Long taskID,
+ @RequestParam(value = "engineType", required = false) String engineType,
+ @RequestParam(value = "creator", required = false) String creator)
+ throws IOException, QueryException {
+ String username = SecurityFilter.getLoginUsername(req);
+ if (endDate == null) {
+ endDate = System.currentTimeMillis();
+ }
+ if (startDate == null) {
+ startDate = 0L;
+ }
+ if (StringUtils.isEmpty(creator)) {
+ creator = null;
+ }
+ Date sDate = new Date(startDate);
+ Date eDate = new Date(endDate);
+ if (startDate == 0L) {
+ sDate = DateUtils.addDays(eDate, -1);
+ }
+ if (sDate.getTime() == eDate.getTime()) {
+ Calendar instance = Calendar.getInstance();
+ instance.setTimeInMillis(endDate);
+ instance.add(Calendar.DAY_OF_MONTH, 1);
+ eDate = new Date(instance.getTime().getTime());
+ }
+ Integer total =
+ jobHistoryQueryService.countUndoneTasks(
+ username,
+ creator,
+ sDate,
+ eDate,
+ engineType,
+ queryCacheManager.getUndoneTaskMinId());
+
+ return Message.ok().data(JobRequestConstants.TOTAL_PAGE(), total);
+ }
}
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java
index cd41a0f7f..ff309ebaa 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java
@@ -38,7 +38,9 @@ public interface JobHistoryQueryService {
JobHistory getJobHistoryByIdAndName(Long jobID, String userName);
- List<JobHistory> search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String executionApplicationName, Long startJobId);
+ List<JobHistory> search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String engineType, Long startJobId);
+
+ Integer countUndoneTasks(String username, String creator, Date sDate, Date eDate, String engineType, Long startJobId);
JobHistory searchOne(Long jobId, Date sDate, Date eDate);
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala
index 39972617c..edf868564 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala
@@ -17,6 +17,7 @@
package org.apache.linkis.jobhistory.service.impl
+import com.google.common.cache.{Cache, CacheBuilder}
import com.google.common.collect.Iterables
import org.apache.commons.lang3.StringUtils
import org.apache.commons.lang3.exception.ExceptionUtils
@@ -38,6 +39,7 @@ import org.springframework.stereotype.Service
import java.sql.Timestamp
import java.{lang, util}
import java.util.Date
+import java.util.concurrent.{Callable, TimeUnit}
import scala.collection.JavaConversions._
import scala.collection.JavaConverters.asScalaBufferConverter
@@ -49,8 +51,13 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
private var jobHistoryMapper: JobHistoryMapper = _
@Autowired
private var jobDetailMapper: JobDetailMapper = _
-// @Autowired
-// private var queryCacheService: QueryCacheService = _
+
+ private val unDoneTaskCache: Cache[String, Integer] = CacheBuilder.newBuilder().concurrencyLevel(5)
+ .expireAfterWrite(1, TimeUnit.MINUTES)
+ .initialCapacity(20)
+ .maximumSize(1000)
+ .recordStats()
+ .build()
@Receiver
override def add(jobReqInsert: JobReqInsert): JobRespProtocol = {
@@ -210,11 +217,11 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
}
override def search(jobId: java.lang.Long, username: String, status: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: java.lang.Long): util.List[JobHistory] = {
- import scala.collection.JavaConversions._
+
val split: util.List[String] = if (status != null) status.split(",").toList else null
val result = if (StringUtils.isBlank(creator)) {
jobHistoryMapper.search(jobId, username, split, sDate, eDate, engineType, startJobId)
- } else if(StringUtils.isBlank(username)) {
+ } else if (StringUtils.isBlank(username)) {
val fakeLabel = new UserCreatorLabel
jobHistoryMapper.searchWithCreatorOnly(jobId, username, fakeLabel.getLabelKey, creator, split, sDate, eDate, engineType, startJobId)
} else {
@@ -223,7 +230,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
fakeLabel.setCreator(creator)
val userCreator = fakeLabel.getStringValue
Utils.tryCatch(fakeLabel.valueCheck(userCreator)) {
- t => logger.info("input user or creator is not correct", t)
+ t => info("input user or creator is not correct", t)
throw t
}
jobHistoryMapper.searchWithUserCreator(jobId, username, fakeLabel.getLabelKey, userCreator, split, sDate, eDate, engineType, startJobId)
@@ -231,14 +238,16 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
result
}
+
+
override def getQueryVOList(list: java.util.List[JobHistory]): java.util.List[JobRequest] = {
jobHistory2JobRequest(list)
}
- private def shouldUpdate(oldStatus: String, newStatus: String): Boolean = {
- if(TaskStatus.valueOf(oldStatus) == TaskStatus.valueOf(newStatus)){
+ private def shouldUpdate(oldStatus: String, newStatus: String): Boolean = {
+ if (TaskStatus.valueOf(oldStatus) == TaskStatus.valueOf(newStatus)) {
true
- }else{
+ } else {
TaskStatus.valueOf(oldStatus).ordinal <= TaskStatus.valueOf(newStatus).ordinal && !TaskStatus.isComplete(TaskStatus.valueOf(oldStatus))
}
}
@@ -256,5 +265,44 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
})
}
+ override def countUndoneTasks(username: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: lang.Long): Integer = {
+ val cacheKey = if (StringUtils.isNoneBlank(username, creator, engineType)) "" else {
+ s"${username}_${creator}_${engineType}"
+ }
+ if (StringUtils.isBlank(cacheKey)) {
+ getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId)
+ } else {
+ unDoneTaskCache.get(cacheKey, new Callable[Integer]{
+ override def call(): Integer = {
+ getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId)
+ }
+ })
+ }
+ }
+
+ private def getCountUndoneTasks(username: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: lang.Long): Integer = {
+ val statusList: util.List[String] = new util.ArrayList[String]()
+ statusList.add(TaskStatus.Running.toString)
+ statusList.add(TaskStatus.Inited.toString)
+ statusList.add(TaskStatus.Scheduled.toString)
+
+ val count = if (StringUtils.isBlank(creator)) {
+ jobHistoryMapper.countUndoneTaskNoCreator(username, statusList, sDate, eDate, engineType, startJobId)
+ } else if (StringUtils.isBlank(username)) {
+ val fakeLabel = new UserCreatorLabel
+ jobHistoryMapper.countUndoneTaskWithCreatorOnly(username, fakeLabel.getLabelKey, creator, statusList, sDate, eDate, engineType, startJobId)
+ } else {
+ val fakeLabel = new UserCreatorLabel
+ fakeLabel.setUser(username)
+ fakeLabel.setCreator(creator)
+ val userCreator = fakeLabel.getStringValue
+ Utils.tryCatch(fakeLabel.valueCheck(userCreator)) {
+ t => logger.info("input user or creator is not correct", t)
+ throw t
+ }
+ jobHistoryMapper.countUndoneTaskWithUserCreator(username, fakeLabel.getLabelKey, userCreator, statusList, sDate, eDate, engineType, startJobId)
+ }
+ count
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org