You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2020/07/01 08:31:57 UTC
[kylin] 15/41: KYLIN-4535 Frontend support for query & storage trend
This is an automated email from the ASF dual-hosted git repository.
nju_yaho pushed a commit to tag ebay-3.1.0-release-20200701
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 637d83f5d75dab7eb1ec72868290f8bb14a618d0
Author: Zhong, Yanghong <nj...@apache.org>
AuthorDate: Thu Jun 11 18:16:14 2020 +0800
KYLIN-4535 Frontend support for query & storage trend
---
.../kylin/rest/service/DashboardService.java | 1 +
.../apache/kylin/rest/service/QueryService.java | 1 -
.../apache/kylin/rest/util/SqlCreationUtil.java | 6 ++-
webapp/app/js/controllers/cube.js | 57 +++++++++++++++++++---
webapp/app/js/controllers/dashboard.js | 12 +++++
webapp/app/js/model/cubeConfig.js | 45 ++++++++++++++++-
webapp/app/js/model/dashboardConfig.js | 5 +-
webapp/app/js/services/cubes.js | 3 +-
webapp/app/js/services/dashboard.js | 2 +-
webapp/app/less/app.less | 6 +--
webapp/app/partials/cubes/cube_detail.html | 18 +++++++
webapp/app/partials/dashboard/dashboard.html | 26 ++++++++--
12 files changed, 158 insertions(+), 24 deletions(-)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
index 17d51aa..4792b1b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@ -115,6 +115,7 @@ public class DashboardService extends BasicService {
jobMetrics.increase("avgJobBuildTime", getMetricValue(row.get(1)));
jobMetrics.increase("maxJobBuildTime", getMetricValue(row.get(2)));
jobMetrics.increase("minJobBuildTime", getMetricValue(row.get(3)));
+ jobMetrics.increase("avgJobExpansionRate", getMetricValue(row.get(4)));
}
return jobMetrics;
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 625b688..7d9b481 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -95,7 +95,6 @@ import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
import org.apache.kylin.metadata.querymeta.TableMeta;
import org.apache.kylin.metadata.querymeta.TableMetaWithType;
import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metrics.MetricsManager;
import org.apache.kylin.query.QueryConnection;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.PushDownUtil;
diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
index 4b6acf9..021bb8b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/util/SqlCreationUtil.java
@@ -95,7 +95,8 @@ public class SqlCreationUtil {
public static PrepareSqlRequest createPrepareSqlRequestOfTotalJobMetrics(String projectName, String cubeName,
String startTime, String endTime) {
String[] measures = new String[] { JobMeasureEnum.JOB_COUNT.toSQL(), JobMeasureEnum.AVG_JOB_BUILD_TIME.toSQL(),
- JobMeasureEnum.MAX_JOB_BUILD_TIME.toSQL(), JobMeasureEnum.MIN_JOB_BUILD_TIME.toSQL() };
+ JobMeasureEnum.MAX_JOB_BUILD_TIME.toSQL(), JobMeasureEnum.MIN_JOB_BUILD_TIME.toSQL(),
+ JobMeasureEnum.EXPANSION_RATE.toSQL() };
return createPrepareSqlRequestOfJobMetrics(projectName, cubeName, startTime, endTime, null, measures);
}
@@ -373,7 +374,8 @@ public class SqlCreationUtil {
JOB_COUNT("count(*)"), //
AVG_JOB_BUILD_TIME("avg(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
MAX_JOB_BUILD_TIME("max(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
- MIN_JOB_BUILD_TIME("min(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
+ MIN_JOB_BUILD_TIME("min(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"), //
+ EXPANSION_RATE(getExpansionRateMetric());
private final String sql;
diff --git a/webapp/app/js/controllers/cube.js b/webapp/app/js/controllers/cube.js
index 5664690..89e0ea2 100755
--- a/webapp/app/js/controllers/cube.js
+++ b/webapp/app/js/controllers/cube.js
@@ -125,7 +125,7 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
if (!cube.currentCuboids) {
CubeService.getCurrentCuboids({cubeId: cube.name}, function(data) {
if (data && data.nodeInfos) {
- $scope.createChart(data, 'current');
+ $scope.createPlannerChart(data, 'current');
cube.currentCuboids = data;
} else {
$scope.currentOptions = angular.copy(cubeConfig.chartOptions);
@@ -136,7 +136,7 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
console.error('current cuboid error', e.data);
});
} else {
- $scope.createChart(cube.currentCuboids, 'current');
+ $scope.createPlannerChart(cube.currentCuboids, 'current');
}
};
@@ -151,7 +151,7 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
if (data.nodeInfos.length === 1 && !data.nodeInfos[0].cuboid_id) {
SweetAlert.swal('Loading', 'Please wait a minute, servers are recommending for you', 'success');
} else {
- $scope.createChart(data, 'recommend');
+ $scope.createPlannerChart(data, 'recommend');
cube.recommendCuboids = data;
// update current chart mark delete node gray.
angular.forEach(cube.currentCuboids.nodeInfos, function(nodeInfo) {
@@ -160,7 +160,7 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
nodeInfo.deleted = true;
}
});
- $scope.createChart(cube.currentCuboids, 'current');
+ $scope.createPlannerChart(cube.currentCuboids, 'current');
$scope.currentChart.api.refresh();
}
} else {
@@ -173,7 +173,7 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
console.error('recommend cuboid error', e.data);
});
} else {
- $scope.createChart(cube.recommendCuboids, 'recommend');
+ $scope.createPlannerChart(cube.recommendCuboids, 'recommend');
}
};
@@ -219,11 +219,11 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
};
// transform chart data and customized options.
- $scope.createChart = function(data, type) {
+ $scope.createPlannerChart = function(data, type) {
var chartData = data.treeNode;
if ('current' === type) {
$scope.currentData = [chartData];
- $scope.currentOptions = angular.copy(cubeConfig.baseChartOptions);
+ $scope.currentOptions = angular.copy(cubeConfig.basePlannerChartOptions);
$scope.currentOptions.caption = angular.copy(cubeConfig.currentCaption);
if ($scope.cube.recommendCuboids){
$scope.currentOptions.caption.css['text-align'] = 'right';
@@ -242,7 +242,7 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
$scope.currentOptions.subtitle.text = '[Cuboid Count: ' + data.nodeInfos.length + '] [Row Count: ' + data.totalRowCount + ']';
} else if ('recommend' === type) {
$scope.recommendData = [chartData];
- $scope.recommendOptions = angular.copy(cubeConfig.baseChartOptions);
+ $scope.recommendOptions = angular.copy(cubeConfig.basePlannerChartOptions);
$scope.recommendOptions.caption = angular.copy(cubeConfig.recommendCaption);
$scope.recommendOptions.chart.color = function(d) {
var cuboid = _.find(data.nodeInfos, function(o) { return o.name == d; });
@@ -290,6 +290,47 @@ KylinApp.controller('CubeCtrl', function ($scope, $rootScope, AccessService, Mes
}
}
+ // click trend tab to get trend chart
+ $scope.getOptimizationTrend = function(cube) {
+ $scope.optimizationTrendData = [];
+ $scope.optimizationTrendOptions = angular.copy(cubeConfig.baseOptimizationTrendChartOptions);
+ CubeService.getOptimizationTrend({cubeId: cube.name}, function (trendData) {
+ var i;
+ if (trendData.trendOfQueryLatency.length) {
+ var queryLatency = {
+ key: 'Query Latency'
+ };
+ var queryLatencyValues = [];
+ for (i = 0; i < trendData.trendOfQueryLatency.length; i++) {
+ queryLatencyValues.push({
+ x: (new Date(trendData.timeSequence[i])).getTime(),
+ y: trendData.trendOfQueryLatency[i]
+ });
+ }
+ queryLatency.values = _.sortBy(queryLatencyValues, 'x');
+ $scope.optimizationTrendData.push(queryLatency);
+ }
+ if (trendData.trendOfStorageUsage.length) {
+ var storageUsage = {
+ key: 'Storage Usage',
+ bar: true
+ };
+ var storageUsageValues = [];
+ for (i = 0; i < trendData.trendOfStorageUsage.length; i++) {
+ storageUsageValues.push({
+ x: (new Date(trendData.timeSequence[i])).getTime(),
+ y: trendData.trendOfStorageUsage[i]
+ });
+ }
+ storageUsage.values = _.sortBy(storageUsageValues, 'x');
+ $scope.optimizationTrendData.push(storageUsage);
+ }
+ }, function (e) {
+ SweetAlert.swal('Oops...', 'Failed to get trend data', 'error');
+ console.error('get trend error', e.data);
+ });
+ };
+
// streaming cube status
$scope.getStreamingInfo = function(cube) {
AdminStreamingService.getCubeRealTimeStats({cubeName: cube.name}, function(data){
diff --git a/webapp/app/js/controllers/dashboard.js b/webapp/app/js/controllers/dashboard.js
index f27ceb0..9d123b5 100644
--- a/webapp/app/js/controllers/dashboard.js
+++ b/webapp/app/js/controllers/dashboard.js
@@ -255,6 +255,18 @@ KylinApp.controller('DashboardCtrl', function ($scope, $location, storage, kylin
$scope.createCharts();
};
+ // Click job expansion rate
+ $scope.jobExpansionChart = function() {
+ $scope.currentSquare = 'jobExpansionRate';
+ $scope.barchartCategory = dashboardConfig.categories[1];
+ $scope.barchartMetric = dashboardConfig.metrics[4];
+ $scope.linechartCategory = dashboardConfig.categories[1];
+ $scope.linechartMetric = dashboardConfig.metrics[4];
+
+ $scope.removeChart('all');
+ $scope.createCharts();
+ };
+
// Line chart granularity change.
$scope.changeDimensionFilter = function(chartType) {
if (chartType === 'line') {
diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js
index 9dc9a68..43e8fed 100644
--- a/webapp/app/js/model/cubeConfig.js
+++ b/webapp/app/js/model/cubeConfig.js
@@ -128,7 +128,7 @@ KylinApp.constant('cubeConfig', {
{name: 'Meta Store', value: 'metaStore'},
{name: 'HBase', value: 'hbase'}
],
- baseChartOptions: {
+ basePlannerChartOptions: {
chart: {
type: 'sunburstChart',
height: 500,
@@ -197,5 +197,48 @@ KylinApp.constant('cubeConfig', {
'text-align': 'left',
'left': '93px'
}
+ },
+ baseOptimizationTrendChartOptions: {
+ chart: {
+ type: 'linePlusBarChart',
+ height: 600,
+ duration: 500,
+ focusEnable: false,
+ margin : {
+ top: 60,
+ right: 100,
+ bottom: 60,
+ left: 100
+ },
+ xAxis: {
+ axisLabel: "Date",
+ tickFormat: function(d) {
+ return d3.time.format('%Y-%m-%d %H:%M:%S')(new Date(d));
+ },
+ },
+ y1Axis: {
+ axisLabel: 'Expansion Rate',
+ tickFormat: function(d) {
+ return d3.format('.2f')(d);
+ },
+ showMaxMin: false
+ },
+ y2Axis: {
+ axisLabel: 'Query Latency (ms)',
+ tickFormat: function(d) {
+ if (d < 1000) {
+ if (parseFloat(d) === d) {
+ return d3.format('.1')(d);
+ } else {
+ return d3.format('.2f')(d);
+ }
+ } else {
+ var prefix = d3.formatPrefix(d);
+ return prefix.scale(d) + prefix.symbol;
+ }
+ },
+ showMaxMin: false
+ },
+ }
}
});
diff --git a/webapp/app/js/model/dashboardConfig.js b/webapp/app/js/model/dashboardConfig.js
index 1620d1d..21c1f85 100644
--- a/webapp/app/js/model/dashboardConfig.js
+++ b/webapp/app/js/model/dashboardConfig.js
@@ -26,7 +26,8 @@ KylinApp.constant('dashboardConfig', {
{name: 'query count', value: 'QUERY_COUNT'},
{name: 'avg query latency', value: 'AVG_QUERY_LATENCY'},
{name: 'job count', value: 'JOB_COUNT'},
- {name: 'avg build time', value: 'AVG_JOB_BUILD_TIME'}
+ {name: 'avg build time', value: 'AVG_JOB_BUILD_TIME'},
+ {name: 'avg expansion rate', value: 'EXPANSION_RATE'}
],
dimensions: [
{name: 'project', value: 'PROJECT'},
@@ -93,4 +94,4 @@ KylinApp.constant('dashboardConfig', {
}
}
}
-});
\ No newline at end of file
+});
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 13de034..f625d67 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -95,6 +95,7 @@ KylinApp.factory('CubeService', ['$resource', function ($resource, config) {
checkDuplicateCubeName: {method: 'GET', params: {action: 'validate'}, isArray: false},
migrate: {method: 'PUT', params: {action: 'migrateRequest'}, isArray: false},
approve: {method: 'PUT', params: {action: 'migrateApprove'}, isArray: false},
- reject: {method: 'PUT', params: {action: 'migrateReject'}, isArray: false}
+ reject: {method: 'PUT', params: {action: 'migrateReject'}, isArray: false},
+ getOptimizationTrend: {method: 'GET', params: {propName: 'optimization', action: 'trend'}, isArray: false}
});
}]);
diff --git a/webapp/app/js/services/dashboard.js b/webapp/app/js/services/dashboard.js
index 4340119..be39bea 100644
--- a/webapp/app/js/services/dashboard.js
+++ b/webapp/app/js/services/dashboard.js
@@ -57,7 +57,7 @@ KylinApp.factory('DashboardService', ['$resource', '$location', function ($resou
var data = response.data;
var jobMetrics;
if (data) {
- jobMetrics = {jobCount: data.jobCount, buildingTime: {avg: data.avgJobBuildTime*1024*1024/1000, max: data.maxJobBuildTime*1024*1024/1000, min: data.minJobBuildTime*1024*1024/1000}};
+ jobMetrics = {jobCount: data.jobCount, buildingTime: {avg: data.avgJobBuildTime*1024*1024/1000, max: data.maxJobBuildTime*1024*1024/1000, min: data.minJobBuildTime*1024*1024/1000}, expansionRate: data.avgJobExpansionRate};
}
return jobMetrics;
}
diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less
index 1b62294..708d430 100644
--- a/webapp/app/less/app.less
+++ b/webapp/app/less/app.less
@@ -878,13 +878,13 @@ pre {
.square {
border: 2px solid #ddd;
text-align: center;
- width: 215px;
+ width: 100%;
height: 170px;
cursor: zoom-in;
padding-top: 15px;
padding-bottom: 15px;
.title {
- font-size: 22px;
+ font-size: 18px;
height: 60px;
}
.metric {
@@ -1170,4 +1170,4 @@ tags-input .tags .tag-item {
}
.pagination{
cursor: pointer;
-}
\ No newline at end of file
+}
diff --git a/webapp/app/partials/cubes/cube_detail.html b/webapp/app/partials/cubes/cube_detail.html
index b747e84..fec8f17 100755
--- a/webapp/app/partials/cubes/cube_detail.html
+++ b/webapp/app/partials/cubes/cube_detail.html
@@ -44,6 +44,9 @@
<li class="{{cube.visiblePage=='planner'? 'active':''}}" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission('cube', cube, permissions.ADMINISTRATION.mask) && !newAccess) && isShowCubeplanner">
<a href="" ng-click="cube.visiblePage='planner';getCubePlanner(cube);">Planner</a>
</li>
+ <li class="{{cube.visiblePage=='trend'? 'active':''}}" ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask)">
+ <a href="" ng-click="cube.visiblePage='trend';getOptimizationTrend(cube);">Trend</a>
+ </li>
<li class="{{cube.visiblePage=='streaming'? 'active':''}}" ng-if="(userService.hasRole('ROLE_ADMIN') || hasPermission('cube', cube, permissions.ADMINISTRATION.mask) && !newAccess) && cube.streamingV2">
<a href="" ng-click="cube.visiblePage='streaming';getStreamingInfo(cube)">Streaming</a>
</li>
@@ -176,6 +179,21 @@
</div>
</div>
+ <div class="cube-detail" ng-if="cube.visiblePage=='trend'">
+ <div style="margin: 15px;">
+ <div ng-if="cube.cuboid_optimized_timestamp_serial.length">
+ <div class="row">
+ <div class="col-md-12">
+ <nvd3 options="optimizationTrendOptions" data="optimizationTrendData"></nvd3>
+ </div>
+ </div>
+ </div>
+ <div ng-if="cube.cuboid_optimized_timestamp_serial.length == 0">
+ <h5>No Optimization Trend Info.</h5>
+ </div>
+ </div>
+ </div>
+
<div class="cube-detail" ng-if="cube.visiblePage=='streaming'">
<div style="padding: 15px; color: #212121;">
<div class="row" ng-repeat="(rsId, replicaSet) in replicaSets" style="padding-left: 10px; padding-right: 10px;">
diff --git a/webapp/app/partials/dashboard/dashboard.html b/webapp/app/partials/dashboard/dashboard.html
index 0d46098..3804509 100644
--- a/webapp/app/partials/dashboard/dashboard.html
+++ b/webapp/app/partials/dashboard/dashboard.html
@@ -62,7 +62,7 @@
</div>
<div class="col-sm-10">
<div class="row">
- <div class="col-sm-3">
+ <div class="col-sm-2" style="width:20%">
<div class="square" ng-class="{'square-active': currentSquare ==='queryCount'}" ng-click="queryCountChart()">
<div class="title">
QUERY<br/>COUNT
@@ -78,7 +78,7 @@
</a>
</div>
</div>
- <div class="col-sm-3">
+ <div class="col-sm-2" style="width:20%">
<div class="square" ng-class="{'square-active': currentSquare ==='queryAvg'}" tooltip-placement="bottom" tooltip="Max: {{queryMetrics.queryLatency.max ? (queryMetrics.queryLatency.max | number:2) : '--'}} sec | Min: {{queryMetrics.queryLatency.min ? (queryMetrics.queryLatency.min| number:2) : '--'}} sec" ng-click="queryAvgChart()">
<div class="title">
AVG QUERY LATENCY
@@ -94,7 +94,7 @@
</a>
</div>
</div>
- <div class="col-sm-3">
+ <div class="col-sm-2" style="width:20%">
<div class="square" ng-class="{'square-active': currentSquare ==='jobCount'}" ng-click="jobCountChart()">
<div class="title">
JOB<br/>COUNT
@@ -110,7 +110,7 @@
</a>
</div>
</div>
- <div class="col-sm-3" >
+ <div class="col-sm-2" style="width:20%" >
<div class="square" ng-class="{'square-active': currentSquare ==='jobBuildTime'}" tooltip-placement="bottom" tooltip="Max: {{jobMetrics.buildingTime.max ? (jobMetrics.buildingTime.max | number:2) : '--'}} sec | Min: {{jobMetrics.buildingTime.min ? ( jobMetrics.buildingTime.min | number:2) : '--'}} sec" ng-click="jobBuildTimeChart()">
<div class="title">
AVG BUILD TIME PER MB
@@ -126,11 +126,27 @@
</a>
</div>
</div>
+ <div class="col-sm-2" style="width:20%">
+ <div class="square" ng-class="{'square-active': currentSquare ==='jobExpansionRate'}" ng-click="jobExpansionChart()">
+ <div class="title">
+ AVG BUILD EXPANSION RATE
+ </div>
+ <div class="metric" ng-if="jobMetrics.expansionRate || jobMetrics.expansionRate === 0">
+ {{jobMetrics.expansionRate | number:2}}<span class="unit"> sec</span>
+ </div>
+ <div class="metric" ng-if="!jobMetrics.expansionRate && (jobMetrics.expansionRate !== 0)">
+ --
+ </div>
+ <a class="description" ng-href="jobs" ng-click="$event.stopPropagation();">
+ More Details
+ </a>
+ </div>
+ </div>
</div>
<div class="row charts">
<div class="col-sm-6" ng-if="barChart">
<div style="border: 2px solid #ddd; margin-top:15px;">
- <div class="form-group" style="width: 96px; position: absolute; right: 15px; bottom: 265px;">
+ <div class="form-group" style="width: 95px; position: absolute; right: 20px; bottom: 265px;">
Show Value: <input type="checkbox" ng-model="barChart.options.chart.showValues">
</div>
<nvd3 options="barChart.options" data="barChart.data"></nvd3>