You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ka...@apache.org on 2018/01/26 09:29:35 UTC
kylin git commit: KYLIN-2999 One click migrate cube in web
Repository: kylin
Updated Branches:
refs/heads/master 34cbd1813 -> a41a7e7d3
KYLIN-2999 One click migrate cube in web
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a41a7e7d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a41a7e7d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a41a7e7d
Branch: refs/heads/master
Commit: a41a7e7d3a97197a16360ab41e94d9ee5c77ebc6
Parents: 34cbd18
Author: kangkaisen <ka...@meituan.com>
Authored: Tue Dec 5 20:37:08 2017 +0800
Committer: kangkaisen <ka...@meituan.com>
Committed: Fri Jan 26 17:28:54 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 25 +++++++++++++
.../kylin/rest/controller/CubeController.java | 7 ++++
.../apache/kylin/rest/service/CubeService.java | 39 ++++++++++++++++++++
webapp/app/js/controllers/cubes.js | 33 ++++++++++++++++-
webapp/app/js/services/cubes.js | 3 +-
webapp/app/js/services/kylinProperties.js | 14 ++++++-
webapp/app/partials/cubes/cubes.html | 1 +
7 files changed, 117 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index cfe6fe2..4c8a3f8 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1572,4 +1572,29 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.metrics.subject-query-rpc", "METRICS_QUERY_RPC") + "_"
+ getKylinMetricsSubjectSuffix();
}
+
+ // ============================================================================
+ // tool
+ // ============================================================================
+ public boolean isAllowAutoMigrateCube() {
+ return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.enabled", "false"));
+ }
+
+ public boolean isAutoMigrateCubeCopyAcl() {
+ return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.copy-acl", "true"));
+ }
+
+ public boolean isAutoMigrateCubePurge() {
+ return Boolean.parseBoolean(getOptional("kylin.tool.auto-migrate-cube.purge-src-cube", "true"));
+ }
+
+ public String getAutoMigrateCubeSrcConfig() {
+ return getOptional("kylin.tool.auto-migrate-cube.src-config", "");
+ }
+
+ public String getAutoMigrateCubeDestConfig() {
+ return getOptional("kylin.tool.auto-migrate-cube.dest-config", "");
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index f6c2abf..89c11a8 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -1022,6 +1022,13 @@ public class CubeController extends BasicController {
}
}
+ @RequestMapping(value = "/{cube}/{project}/migrate", method = { RequestMethod.POST })
+ @ResponseBody
+ public void migrateCube(@PathVariable String cube, @PathVariable String project) {
+ CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cube);
+ cubeService.migrateCube(cubeInstance, project);
+ }
+
public void setCubeService(CubeService cubeService) {
this.cubeService = cubeService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a81d189..154354d 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -30,6 +30,7 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.RootPersistentEntity;
+import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -41,6 +42,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.common.CuboidRecommenderUtil;
+import org.apache.kylin.job.common.PatternedLogger;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
@@ -60,6 +62,7 @@ import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.BadRequestException;
import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
@@ -844,4 +847,40 @@ public class CubeService extends BasicService implements InitializingBean {
Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
}
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public void migrateCube(CubeInstance cube, String projectName) {
+ KylinConfig config = cube.getConfig();
+ try {
+ if (!config.isAllowAutoMigrateCube()) {
+ throw new InternalErrorException("This cube couldn't one click migrate cube, Please contact your ADMIN");
+ }
+
+ for (CubeSegment segment : cube.getSegments()) {
+ if (segment.getStatus() != SegmentStatusEnum.READY) {
+ throw new InternalErrorException("At least one segment is not in READY state. Please check whether there are Running or Error jobs.");
+ }
+ }
+
+ String srcCfgUri = config.getAutoMigrateCubeSrcConfig();
+ String dstCfgUri = config.getAutoMigrateCubeDestConfig();
+
+ String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true");
+ String cmd = String.format(stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri, cube.getName(),
+ projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge());
+
+ logger.info("cmd: " + cmd);
+
+ CliCommandExecutor exec = new CliCommandExecutor();
+ PatternedLogger patternedLogger = new PatternedLogger(logger);
+
+ try {
+ exec.execute(cmd, patternedLogger);
+ } catch (IOException e) {
+ throw new InternalErrorException("Migrating cube fails, Please contact your ADMIN");
+ }
+ } catch (Throwable e) {
+ throw new InternalErrorException(e.getMessage());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index c123f1c..7094ff8 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -293,6 +293,35 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
});
};
+ $scope.isAutoMigrateCubeEnabled = function(){
+ return kylinConfig.isAutoMigrateCubeEnabled();
+ };
+
+ $scope.migrateCube = function (cube) {
+ SweetAlert.swal({
+ title: '',
+ text: "The cube will overwrite the same cube in prod env" +
+ "\nMigrating cube will elapse a couple of minutes." +
+ "\nPlease wait.",
+ type: '',
+ showCancelButton: true,
+ confirmButtonColor: '#DD6B55',
+ confirmButtonText: "Yes",
+ closeOnConfirm: true
+ }, function(isConfirm) {
+ if(isConfirm){
+ loadingRequest.show();
+ CubeService.autoMigrate({cubeId: cube.name, propName: $scope.projectModel.selectedProject}, {}, function (result) {
+ loadingRequest.hide();
+ SweetAlert.swal('Success!', cube.name + ' migrate successfully!', 'success');
+ },function(e){
+ loadingRequest.hide();
+ SweetAlert.swal('Migrate failed!', "Please contact your ADMIN.", 'error');
+ });
+ }
+ });
+ };
+
$scope.startJobSubmit = function (cube) {
$scope.loadDetail(cube).then(function () {
@@ -464,8 +493,6 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
});
});
}
-
-
$scope.cubeEdit = function (cube) {
$location.path("cubes/edit/" + cube.name);
}
@@ -494,6 +521,8 @@ KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location,
});
})
}
+
+
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/js/services/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js
index 150d932..30c2a3f 100644
--- a/webapp/app/js/services/cubes.js
+++ b/webapp/app/js/services/cubes.js
@@ -77,6 +77,7 @@ KylinApp.factory('CubeService', ['$resource', function ($resource, config) {
}
}
},
- optimize: {method: 'PUT', params: {action: 'optimize'}, isArray: false}
+ optimize: {method: 'PUT', params: {action: 'optimize'}, isArray: false},
+ autoMigrate: {method: 'POST', params: {action: 'migrate'}, isArray: false}
});
}]);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/js/services/kylinProperties.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/kylinProperties.js b/webapp/app/js/services/kylinProperties.js
index dc9b94d..ec23397 100644
--- a/webapp/app/js/services/kylinProperties.js
+++ b/webapp/app/js/services/kylinProperties.js
@@ -114,7 +114,7 @@ KylinApp.service('kylinConfig', function (AdminService, $log) {
}
return true;
}
-
+
this.isAdminExportAllowed = function(){
var status = this.getProperty("kylin.web.export-allow-admin").trim();
if(status!=='false'){
@@ -130,12 +130,13 @@ KylinApp.service('kylinConfig', function (AdminService, $log) {
}
return false;
}
-
+
this.getHiddenMeasures = function() {
var hide_measures = this.getProperty("kylin.web.hide-measures").replace(/\s/g,"").toUpperCase();
return hide_measures.split(",")
}
+
this.getQueryTimeout = function () {
var queryTimeout = parseInt(this.getProperty("kylin.web.query-timeout"));
if (isNaN(queryTimeout)) {
@@ -147,5 +148,14 @@ KylinApp.service('kylinConfig', function (AdminService, $log) {
this.isInitialized = function() {
return angular.isString(_config);
}
+
+ this.isAutoMigrateCubeEnabled = function(){
+ var status = this.getProperty("kylin.tool.auto-migrate-cube.enabled").trim();
+ if(status && status =='true'){
+ return true;
+ }
+ return false;
+ }
+
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/a41a7e7d/webapp/app/partials/cubes/cubes.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubes/cubes.html b/webapp/app/partials/cubes/cubes.html
index 94dc3a6..fb98fe3 100644
--- a/webapp/app/partials/cubes/cubes.html
+++ b/webapp/app/partials/cubes/cubes.html
@@ -99,6 +99,7 @@
<li ng-if="cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"><a ng-click="enable(cube)">Enable</a></li>
<li ng-if="cube.status=='DISABLED' && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"><a ng-click="purge(cube)">Purge</a></li>
<li ng-if="cube.status!='DESCBROKEN' && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask))"><a ng-click="cloneCube(cube)">Clone</a></li>
+ <li ng-if="cube.status=='READY' && isAutoMigrateCubeEnabled() && (userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask)) "><a ng-click="migrateCube(cube)">Migrate</a></li>
</ul>
<ul ng-if="!(userService.hasRole('ROLE_ADMIN') || hasPermission('cube',cube, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask))" class="dropdown-menu" role="menu">