You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/10/31 09:11:15 UTC
[kylin] branch 2.5.x updated (a6de1b7 -> c55e904)
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a change to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git.
from a6de1b7 KYLIN-3567 Change scala dependency scope to provided
new 3f0f56a KYLIN-3272 Add spark unregistered class to Kryo
new c55e904 KYLIN-3599 Bulk Add Measures
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../kylin/engine/spark/KylinKryoRegistrator.java | 3 +
webapp/app/js/controllers/cubeMeasures.js | 209 +++++++++++++++++----
webapp/app/partials/cubeDesigner/measures.html | 51 +++++
3 files changed, 225 insertions(+), 38 deletions(-)
[kylin] 02/02: KYLIN-3599 Bulk Add Measures
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit c55e90475cf60338a605d66f0923e94aa77232a9
Author: liapan <li...@ebay.com>
AuthorDate: Thu Sep 27 18:26:38 2018 +0800
KYLIN-3599 Bulk Add Measures
---
webapp/app/js/controllers/cubeMeasures.js | 209 ++++++++++++++++++++-----
webapp/app/partials/cubeDesigner/measures.html | 51 ++++++
2 files changed, 222 insertions(+), 38 deletions(-)
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 6fb82f2..7beb528 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -383,52 +383,116 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
}
}
if($scope.newMeasure.function.parameter.type=="column"&&$scope.newMeasure.function.expression!=="COUNT_DISTINCT"){
+ $scope.newMeasure.function.returntype = $scope.getReturnType($scope.newMeasure.function.parameter.value, $scope.newMeasure.function.expression);
+ }
+ }
+
+ // Open bulk add modal.
+ $scope.openBulkAddModal = function () {
+
+ $scope.initBulkAddMeasures();
+
+ var modalInstance = $modal.open({
+ templateUrl: 'bulkAddMeasures.html',
+ controller: cubeBulkAddMeasureModalCtrl,
+ backdrop: 'static',
+ scope: $scope
+ });
+ };
- var column = $scope.newMeasure.function.parameter.value;
- if(column&&(typeof column=="string")){
- var colType = $scope.getColumnType(VdmUtil.removeNameSpace(column), VdmUtil.getNameSpaceAliasName(column)); // $scope.getColumnType defined in cubeEdit.js
+ $scope.initBulkAddMeasures = function() {
+ // init bulk add measure view model
+ $scope.bulkMeasuresView = {
+ SUM: [],
+ MAX: [],
+ MIN: [],
+ RAW: [],
+ PERCENTILE: []
+ };
+ angular.forEach($scope.getCommonMetricColumns(), function(paramValue) {
+ var measures = _.filter($scope.cubeMetaFrame.measures, function(measure){ return measure.function.parameter.value == paramValue});
+ for (var expression in $scope.bulkMeasuresView) {
+ var bulkMeasure = {
+ name: expression + '_' + paramValue.split('.')[1],
+ parameter: paramValue,
+ returntype: $scope.getReturnType(paramValue, expression),
+ select: false,
+ force: false
+ };
+
+ if (measures.length) {
+ var measure = _.find(measures, function(measure){ return measure.function.expression == expression});
+ if (!!measure) {
+ bulkMeasure.name = measure.name;
+ bulkMeasure.force = true;
+ bulkMeasure.select = true;
+ }
+ }
+ $scope.bulkMeasuresView[expression].push(bulkMeasure);
}
- if(colType==""||!colType){
- $scope.newMeasure.function.returntype = "";
- return;
+ });
+
+ // init expression selector
+ $scope.bulkMeasureOptions = {
+ expressionList: []
+ };
+
+ for (var expression in $scope.bulkMeasuresView) {
+ var selectArr = _.filter($scope.bulkMeasuresView[expression], function(measure){ return measure.select && measure.force});
+ var selectAll = $scope.getCommonMetricColumns().length == selectArr.length;
+ var expressionSelect = {
+ expression: expression,
+ selectAll: selectAll,
+ force: selectAll
}
+ $scope.bulkMeasureOptions.expressionList.push(expressionSelect);
+ }
+ $scope.bulkMeasureOptions.currentExpression = $scope.bulkMeasureOptions.expressionList[0];
+ };
- switch($scope.newMeasure.function.expression){
- case "SUM":
- if(colType==="tinyint"||colType==="smallint"||colType==="int"||colType==="bigint"||colType==="integer"){
- $scope.newMeasure.function.returntype= 'bigint';
+ $scope.getReturnType = function(parameter, expression) {
+ if(parameter && (typeof parameter=="string")){
+ var colType = $scope.getColumnType(VdmUtil.removeNameSpace(parameter), VdmUtil.getNameSpaceAliasName(parameter)); // $scope.getColumnType defined in cubeEdit.js
+ }
+ if(colType == '' || !colType) {
+ return '';
+ }
+
+ switch(expression) {
+ case 'SUM':
+ if(colType === 'tinyint' || colType === 'smallint' || colType === 'int' || colType === 'bigint' || colType === 'integer') {
+ return 'bigint';
+ } else {
+ if(colType.indexOf('decimal') != -1) {
+ var returnRegex = new RegExp('(\\w+)(?:\\((\\w+?)(?:\\,(\\w+?))?\\))?')
+ var returnValue = returnRegex.exec(colType)
+ var precision = 19
+ var scale = returnValue[3]
+ return 'decimal(' + precision + ',' + scale + ')';
}else{
- if(colType.indexOf('decimal')!=-1){
- var returnRegex = new RegExp('(\\w+)(?:\\((\\w+?)(?:\\,(\\w+?))?\\))?')
- var returnValue = returnRegex.exec(colType)
- var precision = 19
- var scale = returnValue[3]
- $scope.newMeasure.function.returntype= 'decimal(' + precision + ',' + scale + ')';
- }else{
- $scope.newMeasure.function.returntype= colType;
- }
+ return colType;
}
- break;
- case "MIN":
- case "MAX":
- $scope.newMeasure.function.returntype = colType;
- break;
- case "RAW":
- $scope.newMeasure.function.returntype = "raw";
- break;
- case "COUNT":
- $scope.newMeasure.function.returntype = "bigint";
- break;
- case "PERCENTILE":
- $scope.newMeasure.function.returntype = "percentile(100)";
- break;
- default:
- $scope.newMeasure.function.returntype = "";
- break;
- }
+ }
+ break;
+ case 'MIN':
+ case 'MAX':
+ return colType;
+ break;
+ case 'RAW':
+ return 'raw';
+ break;
+ case 'COUNT':
+ return 'bigint';
+ break;
+ case 'PERCENTILE':
+ return 'percentile(100)';
+ break;
+ default:
+ return '';
+ break;
}
- }
+ };
if ($scope.state.mode == 'edit') {
$scope.$on('$destroy', function () {
@@ -486,3 +550,72 @@ var NextParameterModalCtrl = function ($scope, scope,para,$modalInstance,cubeCon
}
}
+
+var cubeBulkAddMeasureModalCtrl = function($scope, $modalInstance, SweetAlert) {
+ $scope.cancel = function () {
+ $modalInstance.dismiss('cancel');
+ };
+
+ $scope.saveBulkMeasures = function () {
+ $scope.bulkMeasures = [];
+ // validation
+ loopExp:
+ for (var expression in $scope.bulkMeasuresView) {
+ loopMeasureView:
+ for (var measureView of $scope.bulkMeasuresView[expression]) {
+ if (!measureView.force) {
+ // validation
+ var measureExisted = _.find($scope.cubeMetaFrame.measures, function(measure){ return measure.function.parameter.value == measureView.parameter && measure.function.expression == measureView.expression});
+ if (!!measureExisted) {
+ $scope.bulkMeasures = [];
+ var errMsg = 'Duplicate measure for ' + measureView.name + ' and ' + measureExisted.name + '.';
+ break loopExp;
+ }
+ if (measureView.select) {
+ var measure = {
+ name: measureView.name,
+ function: {
+ expression: expression,
+ returntype: measureView.returntype,
+ parameter: {
+ type: 'column',
+ value: measureView.parameter
+ }
+ }
+ };
+ $scope.bulkMeasures.push(measure);
+ }
+ }
+ }
+ }
+ if (!!errMsg) {
+ SweetAlert.swal('', errMsg, 'warning');
+ } else {
+ $scope.cubeMetaFrame.measures = $scope.cubeMetaFrame.measures.concat($scope.bulkMeasures);
+ $modalInstance.close();
+ }
+ };
+
+ $scope.selectAll = function() {
+ angular.forEach($scope.bulkMeasuresView[$scope.bulkMeasureOptions.currentExpression.expression], function(measure) {
+ if (!measure.force) {
+ measure.select = $scope.bulkMeasureOptions.currentExpression.selectAll;
+ }
+ });
+ };
+
+ $scope.measureChange = function(measureSelect) {
+ if ($scope.bulkMeasureOptions.currentExpression.selectAll) {
+ if (!measureSelect) {
+ $scope.bulkMeasureOptions.currentExpression.selectAll = false;
+ }
+ } else {
+ for(var measureView of $scope.bulkMeasuresView[$scope.bulkMeasureOptions.currentExpression.expression]) {
+ if (!measureView.select) {
+ return;
+ }
+ }
+ $scope.bulkMeasureOptions.currentExpression.selectAll = true;
+ }
+ }
+}
diff --git a/webapp/app/partials/cubeDesigner/measures.html b/webapp/app/partials/cubeDesigner/measures.html
index 848831c..482e17f 100755
--- a/webapp/app/partials/cubeDesigner/measures.html
+++ b/webapp/app/partials/cubeDesigner/measures.html
@@ -20,6 +20,13 @@
<div ng-controller="CubeMeasuresCtrl">
<ng-form name="forms.cube_measure_form">
<div class="dataTables_wrapper form-inline no-footer" ng-if="cubeMetaFrame.measures.length > 0">
+ <div class="row">
+ <div class="col-xs-6" ng-if="state.mode=='edit'">
+ <button type="button" class="btn btn-default" ng-disabled="!metaModel.model.fact_table.length||instance.status=='READY'" ng-click="openBulkAddModal()" >
+ <i class="fa fa-building-o"></i> Bulk Add Measures
+ </button>
+ </div>
+ </div>
<table class="table table-striped table-hover">
<thead>
<tr>
@@ -341,6 +348,50 @@
</ng-form>
</div>
+<script type="text/ng-template" id="bulkAddMeasures.html">
+ <ng-form name="bulk_add_mes_form">
+ <div class="modal-header large-popover">
+ <h4 class="box-title lighter">Bulk Add Measures</h4>
+ <div class="row">
+ <div class="col-xs-12 col-sm-5 btn-group" >
+ <button type="button" class="btn btn-primary" ng-repeat="expressionOpt in bulkMeasureOptions.expressionList" ng-model="bulkMeasureOptions.currentExpression" btn-radio="expressionOpt">{{expressionOpt.expression}}</button>
+ </div>
+ </div>
+ </div>
+ <div class="modal-body">
+ <table class="table table-striped table-hover">
+ <thead>
+ <tr>
+ <th> <input type="checkbox" ng-model="bulkMeasureOptions.currentExpression.selectAll" ng-change="selectAll()" ng-disabled="bulkMeasureOptions.currentExpression.force" /></th>
+ <th>Name</th>
+ <th>Parameters</th>
+ <th>Return Type</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="measure in bulkMeasuresView[bulkMeasureOptions.currentExpression.expression] track by $index">
+ <td>
+ <input type="checkbox" ng-model="measure.select" ng-disabled="measure.force" ng-change="measureChange()"/>
+ </td>
+ <td>
+ <input type="text" ng-model="measure.name" ng-disabled="measure.force" style="width:90%;">
+ </td>
+ <td>
+ <span>{{measure.parameter}}</span>
+ </td>
+ <td>
+ <span>{{measure.returntype}}</span>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <div class="modal-footer">
+ <button class="btn btn-sm btn-info" ng-disabled="bulk_add_mes_form.$invalid" ng-click="saveBulkMeasures()">OK</button>
+ <button class="btn btn-link" ng-click="cancel()">Cancel</button>
+ </div>
+ </ng-form>
+</script>
<script type="text/ng-template" id="expressionTip.html">
<p>All cubes have to contain one measure for Count(1), suggest use "_Count_" as name (Has been generated automatically)</p>
[kylin] 01/02: KYLIN-3272 Add spark unregistered class to Kryo
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 3f0f56a7141a2c8500d72830d042bec4ac157967
Author: shaofengshi <sh...@apache.org>
AuthorDate: Wed Oct 31 17:07:19 2018 +0800
KYLIN-3272 Add spark unregistered class to Kryo
---
.../main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java | 3 +++
1 file changed, 3 insertions(+)
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index a13f96a..ed623d7 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -111,6 +111,9 @@ public class KylinKryoRegistrator implements KryoRegistrator {
addClassQuitely(kyroClasses, "org.apache.spark.sql.catalyst.expressions.GenericInternalRow");
addClassQuitely(kyroClasses, "org.apache.spark.unsafe.types.UTF8String");
+ addClassQuitely(kyroClasses, "org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage");
+ addClassQuitely(kyroClasses, "scala.collection.immutable.Set$EmptySet$");
+
for (Class kyroClass : kyroClasses) {
kryo.register(kyroClass);
}