You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by al...@apache.org on 2022/07/22 12:31:58 UTC

[incubator-linkis] branch dev-1.2.0 updated (6ffcc0ea9 -> f35b758b7)

This is an automated email from the ASF dual-hosted git repository.

alexkun pushed a change to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git


    from 6ffcc0ea9 Update linkis_ddl.sql (#2507)
     new 8265e118f Distributed lock sql optimization close 2503
     new 2f9e9bd9b undonetask interface performance optimization close #2504
     new b59e159f6 Modify the locker to judge the executor status
     new 4b1cb6eb1 Optimized engine startup to support more Retry scenarios close #2505
     new 499ca191b optimize code
     new f35b758b7 update dateFormatHourStd to yyyy-MM-dd HH

The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../linkis/common/variable/DateTypeUtils.scala     |  2 +-
 .../impl/ProcessEngineConnLaunchService.scala      |  4 ++
 .../execution/AccessibleEngineConnExecution.scala  |  1 +
 .../service/EngineConnTimedLockService.scala       | 10 +++-
 .../engine/DefaultEngineCreateService.scala        | 10 ++++
 .../rm/service/impl/DefaultResourceManager.scala   |  5 +-
 .../linkis/manager/dao/LockManagerMapper.java      |  6 ++
 .../impl/DefaultLabelManagerPersistence.java       |  2 +
 .../impl/DefaultLockManagerPersistence.java        |  8 ++-
 .../physical/CodeLogicalUnitExecTask.scala         |  2 +-
 .../linkis/jobhistory/dao/JobHistoryMapper.java    | 38 +++++++++++--
 .../jobhistory/dao/impl/JobHistoryMapper.xml       | 40 ++++++++++++++
 .../jobhistory/restful/api/QueryRestfulApi.java    | 49 ++++++++++++++++-
 .../jobhistory/service/JobHistoryQueryService.java |  4 +-
 .../service/impl/JobHistoryQueryServiceImpl.scala  | 64 +++++++++++++++++++---
 15 files changed, 222 insertions(+), 23 deletions(-)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org


[incubator-linkis] 04/06: Optimized engine startup to support more Retry scenarios close #2505

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alexkun pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git

commit 4b1cb6eb11e0aa1bc311d7931c1d32595752201c
Author: peacewong <wp...@gmail.com>
AuthorDate: Fri Jul 22 19:36:31 2022 +0800

    Optimized engine startup to support more Retry scenarios close #2505
---
 .../server/service/impl/ProcessEngineConnLaunchService.scala   |  4 ++++
 .../manager/am/service/engine/DefaultEngineCreateService.scala | 10 ++++++++++
 .../manager/rm/service/impl/DefaultResourceManager.scala       |  5 +++--
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
index ca0db8b8f..1f004f777 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
@@ -36,6 +36,7 @@ import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLau
 import org.apache.linkis.rpc.Sender
 import org.apache.commons.io.IOUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
+import org.apache.commons.lang3.StringUtils
 import org.apache.linkis.ecm.server.service.LocalDirsHandleService
 import org.apache.linkis.manager.label.utils.LabelUtil
 
@@ -65,6 +66,9 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe
             case ecmError: ECMErrorException =>
               if (ECMErrorCode.EC_START_TIME_OUT == ecmError.getErrCode) {
                 true
+              } else if (StringUtils.isBlank(ecmError.getDesc)) {
+                logger.info("exception desc is null, can be retry")
+                true
               } else {
                 false
               }
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala
index 4dcc236be..733cfb1e5 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineCreateService.scala
@@ -215,6 +215,16 @@ class DefaultEngineCreateService extends AbstractEngineService with EngineCreate
 
     Utils.tryCatch(getEngineNodeManager.updateEngineNode(oldServiceInstance, engineNode)) { t =>
       logger.warn(s"Failed to update engineNode $engineNode", t)
+      val stopEngineRequest = new EngineStopRequest(engineNode.getServiceInstance, ManagerUtils.getAdminUser)
+      engineStopService.asyncStopEngine(stopEngineRequest)
+      val failedEcNode = getEngineNodeManager.getEngineNode(oldServiceInstance)
+      if (null == failedEcNode) {
+        logger.info(s" engineConn is not exists in db: $oldServiceInstance ")
+      } else {
+        failedEcNode.setLabels(nodeLabelService.getNodeLabels(oldServiceInstance))
+        failedEcNode.getLabels.addAll(LabelUtils.distinctLabel(labelFilter.choseEngineLabel(labelList), emNode.getLabels))
+        engineStopService.engineConnInfoClear(failedEcNode)
+      }
       throw new LinkisRetryException(AMConstant.EM_ERROR_CODE, s"Failed to update engineNode: ${t.getMessage}")
     }
 
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala
index 5b675f508..b98a70446 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala
@@ -19,6 +19,7 @@ package org.apache.linkis.manager.rm.service.impl
 
 import com.google.common.collect.Lists
 import org.apache.linkis.common.ServiceInstance
+import org.apache.linkis.common.exception.LinkisRetryException
 import org.apache.linkis.common.utils.{Logging, Utils}
 import org.apache.linkis.governance.common.conf.GovernanceCommonConf
 import org.apache.linkis.manager.common.conf.RMConfiguration
@@ -587,10 +588,10 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ
   private def tryLock(labelContainer: RMLabelContainer, timeOut: Long = -1): Unit = {
     labelContainer.getResourceLabels.foreach {
       case label: Label[_] =>
-        labelContainer.setCurrentLabel(label.asInstanceOf[Label[_]])
+        labelContainer.setCurrentLabel(label)
         val locked = resourceLockService.tryLock(labelContainer, timeOut)
         if (!locked) {
-          throw new RMWarnException(110022, s"try to lock resource label ${labelContainer.getCurrentLabel} over $timeOut ms, please wait a moment and try again!")
+          throw new LinkisRetryException(110022, s"try to lock resource label ${labelContainer.getCurrentLabel} over $timeOut ms, please wait a moment and try again!")
         }
       case _ =>
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org


[incubator-linkis] 02/06: undonetask interface performance optimization close #2504

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alexkun pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git

commit 2f9e9bd9b9309a154dbf04d124e5c9a156f12d76
Author: peacewong <wp...@gmail.com>
AuthorDate: Fri Jul 22 16:30:49 2022 +0800

    undonetask interface performance optimization close #2504
---
 .../impl/DefaultLabelManagerPersistence.java       |  2 +-
 .../linkis/jobhistory/dao/JobHistoryMapper.java    | 38 +++++++++++--
 .../jobhistory/dao/impl/JobHistoryMapper.xml       | 40 ++++++++++++++
 .../jobhistory/restful/api/QueryRestfulApi.java    | 49 ++++++++++++++++-
 .../jobhistory/service/JobHistoryQueryService.java |  4 +-
 .../service/impl/JobHistoryQueryServiceImpl.scala  | 64 +++++++++++++++++++---
 6 files changed, 180 insertions(+), 17 deletions(-)

diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
index 2e4798099..da46fa111 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
+++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
@@ -139,7 +139,7 @@ public class DefaultLabelManagerPersistence implements LabelManagerPersistence {
 
     @Override
     public PersistenceLabel getLabel(int id) {
-        PersistenceLabel persistenceLabel =  labelManagerMapper.getLabel(id);
+        PersistenceLabel persistenceLabel = labelManagerMapper.getLabel(id);
         PersistenceUtils.setValue(persistenceLabel);
         return labelManagerMapper.getLabel(id);
     }
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java
index 12214723e..3c72f6e96 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java
@@ -32,22 +32,22 @@ public interface JobHistoryMapper {
 
     void updateJobHistory(JobHistory jobReq);
 
-    List<JobHistory> search(
+    List<JobHistory> searchWithIdOrderAsc(
             @Param("id") Long id,
             @Param("umUser") String username,
             @Param("status") List<String> status,
             @Param("startDate") Date startDate,
             @Param("endDate") Date endDate,
-            @Param("engineType") String engineType,
-            @Param("startId") Long startId);
+            @Param("engineType") String engineType);
 
-    List<JobHistory> searchWithIdOrderAsc(
+    List<JobHistory> search(
             @Param("id") Long id,
             @Param("umUser") String username,
             @Param("status") List<String> status,
             @Param("startDate") Date startDate,
             @Param("endDate") Date endDate,
-            @Param("engineType") String engineType);
+            @Param("engineType") String engineType,
+            @Param("startId") Long startId);
 
     List<JobHistory> searchWithUserCreator(
             @Param("id") Long id,
@@ -71,5 +71,33 @@ public interface JobHistoryMapper {
             @Param("engineType") String engineType,
             @Param("startId") Long startId);
 
+    Integer countUndoneTaskNoCreator(
+            @Param("umUser") String username,
+            @Param("status") List<String> status,
+            @Param("startDate") Date startDate,
+            @Param("endDate") Date endDate,
+            @Param("engineType") String engineType,
+            @Param("startId") Long startId);
+
+    Integer countUndoneTaskWithUserCreator(
+            @Param("umUser") String username,
+            @Param("userCreatorKey") String userCreatorKey,
+            @Param("userCreatorValue") String userCreator,
+            @Param("status") List<String> status,
+            @Param("startDate") Date startDate,
+            @Param("endDate") Date endDate,
+            @Param("engineType") String engineType,
+            @Param("startId") Long startId);
+
+    Integer countUndoneTaskWithCreatorOnly(
+            @Param("umUser") String username,
+            @Param("userCreatorKey") String userCreatorKey,
+            @Param("creator") String userCreator,
+            @Param("status") List<String> status,
+            @Param("startDate") Date startDate,
+            @Param("endDate") Date endDate,
+            @Param("engineType") String engineType,
+            @Param("startId") Long startId);
+
     String selectJobHistoryStatusForUpdate(Long jobId);
 }
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml
index 2a2823c1f..9dc88cd2e 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml
@@ -173,4 +173,44 @@
         WHERE
         id = #{jobId} FOR UPDATE
     </select>
+
+    <select id="countUndoneTaskNoCreator" useCache="true" resultType="java.lang.Integer" >
+        /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history
+        <where>
+            <if test="startId != null"> id >= #{startId}</if>
+            <if test="umUser != null">and submit_user = #{umUser}</if>
+            <if test="engineType != null">and engine_type = #{engineType}</if>
+            <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if>
+            <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if>
+        </where>
+    </select>
+
+    <select id="countUndoneTaskWithUserCreator" useCache="true" resultType="java.lang.Integer" >
+        /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history
+        <where>
+            <if test="startId != null"> id >= #{startId}</if>
+            <if test="umUser != null">and submit_user = #{umUser}</if>
+            <if test="engineType != null">and engine_type = #{engineType}</if>
+            <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if>
+            <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if>
+            <if test="userCreatorKey != null and userCreatorValue != null">
+                and  LOCATE('"${userCreatorKey}":"${userCreatorValue}', labels) > 0
+            </if>
+        </where>
+    </select>
+
+    <select id="countUndoneTaskWithCreatorOnly" useCache="true" resultType="java.lang.Integer" >
+        /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history
+        <where>
+            <if test="startId != null"> id >= #{startId}</if>
+            <if test="umUser != null">and submit_user = #{umUser}</if>
+            <if test="engineType != null">and engine_type = #{engineType}</if>
+            <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if>
+            <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if>
+            <if test="userCreatorKey != null and creator != null">
+                and labels like '%"${userCreatorKey}":"%-${creator}%'
+            </if>
+        </where>
+    </select>
+
 </mapper>
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java
index 117b92bb1..89bb49f26 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java
@@ -209,8 +209,8 @@ public class QueryRestfulApi {
     }
 
     /** Method list should not contain subjob, which may cause performance problems. */
-    @RequestMapping(path = "/listundone", method = RequestMethod.GET)
-    public Message listundone(
+    @RequestMapping(path = "/listundonetasks", method = RequestMethod.GET)
+    public Message listundonetasks(
             HttpServletRequest req,
             @RequestParam(value = "startDate", required = false) Long startDate,
             @RequestParam(value = "endDate", required = false) Long endDate,
@@ -281,4 +281,49 @@ public class QueryRestfulApi {
                 .data(TaskConstant.TASKS, vos)
                 .data(JobRequestConstants.TOTAL_PAGE(), total);
     }
+
+    /** Method list should not contain subjob, which may cause performance problems. */
+    @RequestMapping(path = "/listundone", method = RequestMethod.GET)
+    public Message listundone(
+            HttpServletRequest req,
+            @RequestParam(value = "startDate", required = false) Long startDate,
+            @RequestParam(value = "endDate", required = false) Long endDate,
+            @RequestParam(value = "pageNow", required = false) Integer pageNow,
+            @RequestParam(value = "pageSize", required = false) Integer pageSize,
+            @RequestParam(value = "startTaskID", required = false) Long taskID,
+            @RequestParam(value = "engineType", required = false) String engineType,
+            @RequestParam(value = "creator", required = false) String creator)
+            throws IOException, QueryException {
+        String username = SecurityFilter.getLoginUsername(req);
+        if (endDate == null) {
+            endDate = System.currentTimeMillis();
+        }
+        if (startDate == null) {
+            startDate = 0L;
+        }
+        if (StringUtils.isEmpty(creator)) {
+            creator = null;
+        }
+        Date sDate = new Date(startDate);
+        Date eDate = new Date(endDate);
+        if (startDate == 0L) {
+            sDate = DateUtils.addDays(eDate, -1);
+        }
+        if (sDate.getTime() == eDate.getTime()) {
+            Calendar instance = Calendar.getInstance();
+            instance.setTimeInMillis(endDate);
+            instance.add(Calendar.DAY_OF_MONTH, 1);
+            eDate = new Date(instance.getTime().getTime());
+        }
+        Integer total =
+                jobHistoryQueryService.countUndoneTasks(
+                        username,
+                        creator,
+                        sDate,
+                        eDate,
+                        engineType,
+                        queryCacheManager.getUndoneTaskMinId());
+
+        return Message.ok().data(JobRequestConstants.TOTAL_PAGE(), total);
+    }
 }
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java
index cd41a0f7f..ff309ebaa 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java
@@ -38,7 +38,9 @@ public interface JobHistoryQueryService {
 
     JobHistory getJobHistoryByIdAndName(Long jobID, String userName);
 
-    List<JobHistory> search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String executionApplicationName, Long startJobId);
+    List<JobHistory> search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String engineType, Long startJobId);
+
+    Integer countUndoneTasks(String username, String creator, Date sDate, Date eDate, String engineType, Long startJobId);
 
     JobHistory searchOne(Long jobId, Date sDate, Date eDate);
 
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala
index 39972617c..edf868564 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala
+++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala
@@ -17,6 +17,7 @@
  
 package org.apache.linkis.jobhistory.service.impl
 
+import com.google.common.cache.{Cache, CacheBuilder}
 import com.google.common.collect.Iterables
 import org.apache.commons.lang3.StringUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
@@ -38,6 +39,7 @@ import org.springframework.stereotype.Service
 import java.sql.Timestamp
 import java.{lang, util}
 import java.util.Date
+import java.util.concurrent.{Callable, TimeUnit}
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters.asScalaBufferConverter
 
@@ -49,8 +51,13 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
   private var jobHistoryMapper: JobHistoryMapper = _
   @Autowired
   private var jobDetailMapper: JobDetailMapper = _
-//  @Autowired
-//  private var queryCacheService: QueryCacheService = _
+
+  private val unDoneTaskCache: Cache[String, Integer] = CacheBuilder.newBuilder().concurrencyLevel(5)
+    .expireAfterWrite(1, TimeUnit.MINUTES)
+    .initialCapacity(20)
+    .maximumSize(1000)
+    .recordStats()
+    .build()
 
   @Receiver
   override def add(jobReqInsert: JobReqInsert): JobRespProtocol = {
@@ -210,11 +217,11 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
   }
 
   override def search(jobId: java.lang.Long, username: String, status: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: java.lang.Long): util.List[JobHistory] = {
-    import scala.collection.JavaConversions._
+
     val split: util.List[String] = if (status != null) status.split(",").toList else null
     val result = if (StringUtils.isBlank(creator)) {
       jobHistoryMapper.search(jobId, username, split, sDate, eDate, engineType, startJobId)
-    } else if(StringUtils.isBlank(username)) {
+    } else if (StringUtils.isBlank(username)) {
       val fakeLabel = new UserCreatorLabel
       jobHistoryMapper.searchWithCreatorOnly(jobId, username, fakeLabel.getLabelKey, creator, split, sDate, eDate, engineType, startJobId)
     } else {
@@ -223,7 +230,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
       fakeLabel.setCreator(creator)
       val userCreator = fakeLabel.getStringValue
       Utils.tryCatch(fakeLabel.valueCheck(userCreator)) {
-        t => logger.info("input user or creator is not correct", t)
+        t => info("input user or creator is not correct", t)
           throw t
       }
       jobHistoryMapper.searchWithUserCreator(jobId, username, fakeLabel.getLabelKey, userCreator, split, sDate, eDate, engineType, startJobId)
@@ -231,14 +238,16 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
     result
   }
 
+
+
   override def getQueryVOList(list: java.util.List[JobHistory]): java.util.List[JobRequest] = {
     jobHistory2JobRequest(list)
   }
 
-  private def shouldUpdate(oldStatus: String, newStatus: String): Boolean =  {
-    if(TaskStatus.valueOf(oldStatus) == TaskStatus.valueOf(newStatus)){
+  private def shouldUpdate(oldStatus: String, newStatus: String): Boolean = {
+    if (TaskStatus.valueOf(oldStatus) == TaskStatus.valueOf(newStatus)) {
       true
-    }else{
+    } else {
       TaskStatus.valueOf(oldStatus).ordinal <= TaskStatus.valueOf(newStatus).ordinal && !TaskStatus.isComplete(TaskStatus.valueOf(oldStatus))
     }
   }
@@ -256,5 +265,44 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging {
       })
   }
 
+  override def countUndoneTasks(username: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: lang.Long): Integer = {
+    val cacheKey = if (StringUtils.isNoneBlank(username, creator, engineType)) "" else {
+      s"${username}_${creator}_${engineType}"
+    }
+    if (StringUtils.isBlank(cacheKey)) {
+      getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId)
+    } else {
+      unDoneTaskCache.get(cacheKey, new Callable[Integer]{
+        override def call(): Integer = {
+          getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId)
+        }
+      })
+    }
+  }
+
+  private  def getCountUndoneTasks(username: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: lang.Long): Integer = {
+    val statusList: util.List[String] = new util.ArrayList[String]()
+    statusList.add(TaskStatus.Running.toString)
+    statusList.add(TaskStatus.Inited.toString)
+    statusList.add(TaskStatus.Scheduled.toString)
+
+    val count = if (StringUtils.isBlank(creator)) {
+      jobHistoryMapper.countUndoneTaskNoCreator(username, statusList, sDate, eDate, engineType, startJobId)
+    } else if (StringUtils.isBlank(username)) {
+      val fakeLabel = new UserCreatorLabel
+      jobHistoryMapper.countUndoneTaskWithCreatorOnly(username, fakeLabel.getLabelKey, creator, statusList, sDate, eDate, engineType, startJobId)
+    } else {
+      val fakeLabel = new UserCreatorLabel
+      fakeLabel.setUser(username)
+      fakeLabel.setCreator(creator)
+      val userCreator = fakeLabel.getStringValue
+      Utils.tryCatch(fakeLabel.valueCheck(userCreator)) {
+        t => logger.info("input user or creator is not correct", t)
+          throw t
+      }
+      jobHistoryMapper.countUndoneTaskWithUserCreator(username, fakeLabel.getLabelKey, userCreator, statusList, sDate, eDate, engineType, startJobId)
+    }
+    count
+  }
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org


[incubator-linkis] 01/06: Distributed lock sql optimization close 2503

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alexkun pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git

commit 8265e118f6e9b23f4cf762325b234a3cb57b9182
Author: peacewong <wp...@gmail.com>
AuthorDate: Fri Jul 22 16:23:59 2022 +0800

    Distributed lock sql optimization close 2503
---
 .../java/org/apache/linkis/manager/dao/LockManagerMapper.java     | 6 ++++++
 .../manager/persistence/impl/DefaultLabelManagerPersistence.java  | 2 ++
 .../manager/persistence/impl/DefaultLockManagerPersistence.java   | 8 +++++++-
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LockManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LockManagerMapper.java
index 0135ee2a5..14cca3028 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LockManagerMapper.java
+++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LockManagerMapper.java
@@ -33,6 +33,12 @@ public interface LockManagerMapper {
     @Delete("delete  from linkis_cg_manager_lock where lock_object = #{jsonObject}")
     void unlock(String jsonObject);
 
+    @Delete("delete  from linkis_cg_manager_lock where id = #{id}")
+    void unlock(int id);
+
+    @Select("select * from  linkis_cg_manager_lock where lock_object = #{jsonObject}")
+    List<PersistenceLock> getLockersByLockObject(String jsonObject);
+
     @Select("select * from  linkis_cg_manager_lock")
     @Results({
         @Result(property = "updateTime", column = "update_time"),
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
index 191882171..2e4798099 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
+++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java
@@ -139,6 +139,8 @@ public class DefaultLabelManagerPersistence implements LabelManagerPersistence {
 
     @Override
     public PersistenceLabel getLabel(int id) {
+        PersistenceLabel persistenceLabel =  labelManagerMapper.getLabel(id);
+        PersistenceUtils.setValue(persistenceLabel);
         return labelManagerMapper.getLabel(id);
     }
 
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java
index 4ce0b9018..240055df6 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java
+++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java
@@ -70,7 +70,13 @@ public class DefaultLockManagerPersistence implements LockManagerPersistence {
 
     @Override
     public void unlock(PersistenceLock persistenceLock) {
-        lockManagerMapper.unlock(persistenceLock.getLockObject());
+        List<PersistenceLock> lockers =
+                lockManagerMapper.getLockersByLockObject(persistenceLock.getLockObject());
+        if (lockers != null && !lockers.isEmpty()) {
+            for (PersistenceLock lock : lockers) {
+                lockManagerMapper.unlock(lock.getId());
+            }
+        }
     }
 
     @Override


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org


[incubator-linkis] 03/06: Modify the locker to judge the executor status

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alexkun pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git

commit b59e159f64db937028267be924b5c1c126ea0137
Author: peacewong <wp...@gmail.com>
AuthorDate: Fri Jul 22 17:31:46 2022 +0800

    Modify the locker to judge the executor status
---
 .../executor/execution/AccessibleEngineConnExecution.scala     |  1 +
 .../executor/service/EngineConnTimedLockService.scala          | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala
index 9dd89b775..a0b203ce3 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala
@@ -98,6 +98,7 @@ class AccessibleEngineConnExecution extends EngineConnExecution with Logging {
             accessibleExecutor.updateLastActivityTime()
           } else {
             logger.warn(s"${accessibleExecutor.getId} has not been used for $maxFreeTimeStr, now try to shutdown it.")
+            accessibleExecutor.tryShutdown()
             requestManagerReleaseExecutor(" idle release")
             ShutdownHook.getShutdownHook.notifyStop()
           }
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
index 86baf54b6..bbbae2c51 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
@@ -89,8 +89,14 @@ class EngineConnTimedLockService extends LockService with Logging {
     // Lock is binded to engineconn, so choose default executor
     ExecutorManager.getInstance.getReportExecutor match {
       case accessibleExecutor: AccessibleExecutor =>
-        logger.debug("try to lock for executor state is " + accessibleExecutor.getStatus)
-        logger.debug("try to lock for executor id is " + accessibleExecutor.getId)
+        if (logger.isDebugEnabled) {
+          logger.debug("try to lock for executor state is " + accessibleExecutor.getStatus)
+          logger.debug("try to lock for executor id is " + accessibleExecutor.getId)
+        }
+        if (NodeStatus.isCompleted(accessibleExecutor.getStatus)) {
+          logger.error(s"Cannot to lock completed ${accessibleExecutor.getStatus} stats executor")
+          return None
+        }
         if (null == engineConnLock) {
           engineConnLock = new EngineConnTimedLock(timeout)
           ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus.addListener(engineConnLock)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org


[incubator-linkis] 06/06: update dateFormatHourStd to yyyy-MM-dd HH

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alexkun pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git

commit f35b758b7ae275b9da68edb5de30f9ad271f592e
Author: peacewong <wp...@gmail.com>
AuthorDate: Fri Jul 22 19:45:38 2022 +0800

    update dateFormatHourStd to yyyy-MM-dd HH
---
 .../main/scala/org/apache/linkis/common/variable/DateTypeUtils.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/DateTypeUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/DateTypeUtils.scala
index cedc8933c..ec4a8a561 100644
--- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/DateTypeUtils.scala
+++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/variable/DateTypeUtils.scala
@@ -43,7 +43,7 @@ object DateTypeUtils {
   }
 
   val dateFormatHourStdLocal = new ThreadLocal[SimpleDateFormat]() {
-    override protected def initialValue = new SimpleDateFormat("yyyy-MM-dd-HH")
+    override protected def initialValue = new SimpleDateFormat("yyyy-MM-dd HH")
   }
 
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org


[incubator-linkis] 05/06: optimize code

Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

alexkun pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git

commit 499ca191bfde097c56f68c46b9e81431d734db55
Author: peacewong <wp...@gmail.com>
AuthorDate: Fri Jul 22 19:38:09 2022 +0800

    optimize code
---
 .../orchestrator/computation/physical/CodeLogicalUnitExecTask.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/physical/CodeLogicalUnitExecTask.scala b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/physical/CodeLogicalUnitExecTask.scala
index 4da60a065..6c17cec7b 100644
--- a/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/physical/CodeLogicalUnitExecTask.scala
+++ b/linkis-orchestrator/linkis-computation-orchestrator/src/main/scala/org/apache/linkis/orchestrator/computation/physical/CodeLogicalUnitExecTask.scala
@@ -91,7 +91,7 @@ class CodeLogicalUnitExecTask (parents: Array[ExecTask], children: Array[ExecTas
           codeExecutor.setEngineConnTaskId(engineConnExecId)
           codeExecTaskExecutorManager.addEngineConnTaskID(codeExecutor)
           val infoMap = new  util.HashMap[String, Object]
-          infoMap.put(TaskConstant.ENGINE_INSTANCE, codeExecutor.getEngineConnExecutor.getServiceInstance)
+          infoMap.put(TaskConstant.ENGINE_INSTANCE, codeExecutor.getEngineConnExecutor.getServiceInstance.getInstance)
           val event = TaskRunningInfoEvent(this, 0f,
             Array.empty, new util.HashMap[String, ResourceWithStatus], infoMap)
           getPhysicalContext.pushProgress(event)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org