You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bo...@apache.org on 2019/09/17 09:34:15 UTC
[kylin] branch master updated: KYLIN-4100: Add overall job number
statistics in monitor page
This is an automated email from the ASF dual-hosted git repository.
boblu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 199bc17 KYLIN-4100: Add overall job number statistics in monitor page
199bc17 is described below
commit 199bc17e572652b8a8ec8c4f0c6aab1a2359c1ef
Author: Liu Shaohui <li...@xiaomi.com>
AuthorDate: Mon Sep 16 19:59:59 2019 +0800
KYLIN-4100: Add overall job number statistics in monitor page
---
.../java/org/apache/kylin/job/JobSearchResult.java | 12 +++++++
.../kylin/engine/mr/common/JobInfoConverter.java | 1 +
.../kylin/rest/controller/JobController.java | 41 ++++++++++++++++++++++
.../org/apache/kylin/rest/service/JobService.java | 23 ++++++++++++
webapp/app/js/controllers/job.js | 26 +++++++++++++-
webapp/app/js/model/jobConfig.js | 14 ++++----
webapp/app/js/model/jobListModel.js | 20 +++++++++--
webapp/app/js/services/jobs.js | 1 +
webapp/app/partials/jobs/jobList.html | 4 +--
9 files changed, 130 insertions(+), 12 deletions(-)
diff --git a/core-job/src/main/java/org/apache/kylin/job/JobSearchResult.java b/core-job/src/main/java/org/apache/kylin/job/JobSearchResult.java
index bd4326d..7275f2e 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JobSearchResult.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JobSearchResult.java
@@ -18,6 +18,8 @@
package org.apache.kylin.job;
+import org.apache.kylin.job.constant.JobStatusEnum;
+
public class JobSearchResult implements Comparable<JobSearchResult> {
private String id;
@@ -27,6 +29,8 @@ public class JobSearchResult implements Comparable<JobSearchResult> {
private long lastModified;
+ private JobStatusEnum jobStatus;
+
public String getId() {
return id;
}
@@ -59,6 +63,14 @@ public class JobSearchResult implements Comparable<JobSearchResult> {
this.lastModified = lastModified;
}
+ public JobStatusEnum getJobStatus() {
+ return jobStatus;
+ }
+
+ public void setJobStatus(JobStatusEnum jobStatus) {
+ this.jobStatus = jobStatus;
+ }
+
@Override
public int compareTo(JobSearchResult o) {
return o.lastModified < this.lastModified ? -1 : o.lastModified > this.lastModified ? 1 : 0;
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
index 53246fa..d3519e8 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/JobInfoConverter.java
@@ -237,6 +237,7 @@ public class JobInfoConverter {
result.setId(job.getId());
result.setJobName(job.getName());
result.setLastModified(output.getLastModified());
+ result.setJobStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
return result;
}
}
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
index d6d612d..29c4621 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -93,6 +93,47 @@ public class JobController extends BasicController {
}
/**
+ * get job status overview
+ *
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/overview", method = { RequestMethod.GET }, produces = { "application/json" })
+ @ResponseBody
+ public Map<JobStatusEnum, Integer> jobOverview(JobListRequest jobRequest) {
+ Map<JobStatusEnum, Integer> jobOverview = new HashMap<>();
+ List<JobStatusEnum> statusList = new ArrayList<JobStatusEnum>();
+ if (null != jobRequest.getStatus()) {
+ for (int status : jobRequest.getStatus()) {
+ statusList.add(JobStatusEnum.getByCode(status));
+ }
+ }
+
+ JobTimeFilterEnum timeFilter = JobTimeFilterEnum.LAST_ONE_WEEK;
+ if (null != jobRequest.getTimeFilter()) {
+ timeFilter = JobTimeFilterEnum.getByCode(jobRequest.getTimeFilter());
+ }
+
+ JobService.JobSearchMode jobSearchMode = JobService.JobSearchMode.CUBING_ONLY;
+ if (null != jobRequest.getJobSearchMode()) {
+ try {
+ jobSearchMode = JobService.JobSearchMode.valueOf(jobRequest.getJobSearchMode());
+ } catch (IllegalArgumentException e) {
+ logger.warn("Invalid value for JobSearchMode: '" + jobRequest.getJobSearchMode() + "', skip it.");
+ }
+ }
+
+ try {
+ jobOverview = jobService.searchJobsOverview(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList,
+ timeFilter, jobSearchMode);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e);
+ }
+ return jobOverview;
+ }
+
+ /**
* Get a cube job
*
* @return
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 82738ab..9a36550 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -24,6 +24,7 @@ import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -994,6 +995,28 @@ public class JobService extends BasicService implements InitializingBean {
return jobInstanceList;
}
+ public Map<JobStatusEnum, Integer> searchJobsOverview(final String cubeNameSubstring, final String projectName,
+ final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter,
+ JobSearchMode jobSearchMode) {
+ // TODO: can be optimized here
+ List<JobSearchResult> jobSearchResultList = searchJobsByCubeNameV2(cubeNameSubstring, projectName, statusList,
+ timeFilter, jobSearchMode);
+ Map<JobStatusEnum, Integer> jobOverview = new HashMap<>();
+ if (statusList == null || statusList.isEmpty()) {
+ for (JobStatusEnum status: JobStatusEnum.values()) {
+ jobOverview.put(status, 0);
+ }
+ } else {
+ for (JobStatusEnum status: statusList) {
+ jobOverview.put(status, 0);
+ }
+ }
+ for (JobSearchResult result : jobSearchResultList) {
+ jobOverview.put(result.getJobStatus(), jobOverview.get(result.getJobStatus()) + 1);
+ }
+ return jobOverview;
+ }
+
/**
* it loads all cache for digest metadata of "execute" and "execute_output", and returns the search results within the scope of the given filters
*
diff --git a/webapp/app/js/controllers/job.js b/webapp/app/js/controllers/job.js
index 6ad7318..1f64efc 100644
--- a/webapp/app/js/controllers/job.js
+++ b/webapp/app/js/controllers/job.js
@@ -110,11 +110,35 @@ KylinApp
SweetAlert.swal('Oops...', resp, 'error');
return defer.promise;
});
- }
+ };
+
+ $scope.overview = function () {
+ var defer = $q.defer();
+ var statusIds = [];
+ angular.forEach($scope.status, function (statusObj, index) {
+ statusIds.push(statusObj.value);
+ });
+ var jobRequest = {
+ cubeName: $scope.cubeName,
+ projectName: $scope.state.projectName,
+ status: statusIds,
+ timeFilter: $scope.timeFilter.value,
+ jobSearchMode: $scope.searchMode.value
+ };
+ return JobList.overview(jobRequest).then(function(resp){
+ defer.resolve(resp);
+ return defer.promise;
+ },function(resp){
+ defer.resolve([]);
+ SweetAlert.swal('Oops...', resp, 'error');
+ return defer.promise;
+ });
+ };
$scope.reload = function () {
// trigger reload action in pagination directive
$scope.action.reload = !$scope.action.reload;
+ $scope.overview()
};
diff --git a/webapp/app/js/model/jobConfig.js b/webapp/app/js/model/jobConfig.js
index d840fb1..f4ec3de 100644
--- a/webapp/app/js/model/jobConfig.js
+++ b/webapp/app/js/model/jobConfig.js
@@ -18,13 +18,13 @@
KylinApp.constant('jobConfig', {
allStatus: [
- {name: 'NEW', value: 0},
- {name: 'PENDING', value: 1},
- {name: 'RUNNING', value: 2},
- {name: 'STOPPED', value: 32},
- {name: 'FINISHED', value: 4},
- {name: 'ERROR', value: 8},
- {name: 'DISCARDED', value: 16}
+ {name: 'NEW', value: 0, count: ''},
+ {name: 'PENDING', value: 1, count: ''},
+ {name: 'RUNNING', value: 2, count: ''},
+ {name: 'STOPPED', value: 32, count: ''},
+ {name: 'FINISHED', value: 4, count: ''},
+ {name: 'ERROR', value: 8, count: ''},
+ {name: 'DISCARDED', value: 16, count: ''}
],
timeFilter: [
{name: 'CURRENT DAY', value: 0},
diff --git a/webapp/app/js/model/jobListModel.js b/webapp/app/js/model/jobListModel.js
index f32fecf..c13ae66 100755
--- a/webapp/app/js/model/jobListModel.js
+++ b/webapp/app/js/model/jobListModel.js
@@ -20,7 +20,7 @@
*jobListModel will manage data in list job page
*/
-KylinApp.service('JobList',function(JobService, $q, kylinConfig){
+KylinApp.service('JobList',function(JobService, $q, kylinConfig, jobConfig){
var _this = this;
this.jobs={};
this.jobFilter = {
@@ -55,7 +55,6 @@ KylinApp.service('JobList',function(JobService, $q, kylinConfig){
}
_this.jobs[id].dropped = false;
});
-
defer.resolve(jobs.length);
},function(){
defer.reject("failed to load jobs");
@@ -63,6 +62,23 @@ KylinApp.service('JobList',function(JobService, $q, kylinConfig){
return defer.promise;
};
+ this.overview = function(jobRequest){
+ var defer = $q.defer();
+ JobService.overview(jobRequest, function (jobsOverview) {
+ angular.forEach(jobConfig.allStatus, function (key) {
+ if (angular.isDefined(jobsOverview[key.name])) {
+ key.count = "(" + jobsOverview[key.name] + ")"
+ } else {
+ key.count = "";
+ }
+ });
+ defer.resolve(jobsOverview);
+ },function(){
+ defer.reject("failed to load job overview");
+ });
+ return defer.promise;
+ };
+
this.removeAll = function(){
_this.jobs={};
};
diff --git a/webapp/app/js/services/jobs.js b/webapp/app/js/services/jobs.js
index 337126a..a0fbcc6 100644
--- a/webapp/app/js/services/jobs.js
+++ b/webapp/app/js/services/jobs.js
@@ -20,6 +20,7 @@ KylinApp.factory('JobService', ['$resource', function ($resource, config) {
return $resource(Config.service.url + 'jobs/:jobId/:propName/:propValue/:action', {}, {
list: {method: 'GET', params: {}, isArray: true},
get: {method: 'GET', params: {}, isArray: false},
+ overview: {method: 'GET', params: {propName: 'overview'}, isArray: false},
stepOutput: {method: 'GET', params: {propName: 'steps', action: 'output'}, isArray: false},
resume: {method: 'PUT', params: {action: 'resume'}, isArray: false},
cancel: {method: 'PUT', params: {action: 'cancel'}, isArray: false},
diff --git a/webapp/app/partials/jobs/jobList.html b/webapp/app/partials/jobs/jobList.html
index ae05590..b72dd94 100644
--- a/webapp/app/partials/jobs/jobList.html
+++ b/webapp/app/partials/jobs/jobList.html
@@ -37,7 +37,7 @@
</div>
<!--<div class="col-xs-2"><label class="table-header-text">Jobs</label></div>-->
- <div class="col-xs-9">
+ <div class="col-xs-12">
<!--STATUS-->
<div class="pull-right">
<!--Job History Time Filter-->
@@ -53,7 +53,7 @@
<input type="checkbox"
value="{{s.name}}"
ng-checked="status.indexOf(s) > -1"
- ng-click="toggleSelection(s);" />{{s.name}}
+ ng-click="toggleSelection(s);" />{{s.name}}{{s.count}}
</label>
<!--Refresh Jobs-->
<button class="btn btn-success btn-xs" style="margin-left: 10px" ng-click="jobList.removeAll();reload();"><i class="fa fa-refresh"></i></button>