You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ch...@apache.org on 2018/02/07 02:27:39 UTC
kylin git commit: KYLIN-2999 fix cube auto-migration
Repository: kylin
Updated Branches:
refs/heads/master eac03cdcd -> c686b8436
KYLIN-2999 fix cube auto-migration
Signed-off-by: chenzhx <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c686b843
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c686b843
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c686b843
Branch: refs/heads/master
Commit: c686b8436369df1fa951a42f179caa72984e1597
Parents: eac03cd
Author: etherge <et...@163.com>
Authored: Tue Jan 30 14:58:06 2018 +0800
Committer: chenzhx <ch...@apache.org>
Committed: Wed Feb 7 10:27:33 2018 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 26 +++++-----
.../apache/kylin/rest/service/CubeService.java | 51 +++++++++++---------
webapp/app/js/controllers/cubes.js | 2 +-
3 files changed, 42 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c686b843/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 7b48935..eee2d09 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -742,7 +742,7 @@ abstract public class KylinConfigBase implements Serializable {
public String getHiveBeelineShell() {
return getOptional("kylin.source.hive.beeline-shell", "beeline");
}
-
+
public String getHiveBeelineParams() {
return getOptional("kylin.source.hive.beeline-params", "");
}
@@ -750,11 +750,11 @@ abstract public class KylinConfigBase implements Serializable {
public boolean getEnableSparkSqlForTableOps() {
return Boolean.parseBoolean(getOptional("kylin.source.hive.enable-sparksql-for-table-ops", "false"));
}
-
+
public String getSparkSqlBeelineShell() {
return getOptional("kylin.source.hive.sparksql-beeline-shell", "");
}
-
+
public String getSparkSqlBeelineParams() {
return getOptional("kylin.source.hive.sparksql-beeline-params", "");
}
@@ -1504,14 +1504,17 @@ abstract public class KylinConfigBase implements Serializable {
}
public String getPropertiesWhiteList() {
- return getOptional("kylin.web.properties.whitelist",
- "kylin.web.timezone,kylin.query.cache-enabled,kylin.env,kylin.web.hive-limit,kylin.storage.default,kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic,"
- + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,kylin.web.help.3,"
- + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,kylin.security.profile,"
- + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other,"
- + "kylin.cube.cubeplanner.enabled,kylin.web.dashboard-enabled");
+ return getOptional("kylin.web.properties.whitelist", "kylin.web.timezone,kylin.query.cache-enabled,kylin.env,"
+ + "kylin.web.hive-limit,kylin.storage.default,"
+ + "kylin.engine.default,kylin.web.link-hadoop,kylin.web.link-diagnostic,"
+ + "kylin.web.contact-mail,kylin.web.help.length,kylin.web.help.0,kylin.web.help.1,kylin.web.help.2,"
+ + "kylin.web.help.3,"
+ + "kylin.web.help,kylin.web.hide-measures,kylin.web.link-streaming-guide,kylin.server.external-acl-provider,"
+ + "kylin.security.profile,"
+ + "kylin.htrace.show-gui-trace-toggle,kylin.web.export-allow-admin,kylin.web.export-allow-other,"
+ + "kylin.cube.cubeplanner.enabled,kylin.web.dashboard-enabled,kylin.tool.auto-migrate-cube.enabled");
}
-
+
// ============================================================================
// RESTCLIENT
// ============================================================================
@@ -1527,7 +1530,7 @@ abstract public class KylinConfigBase implements Serializable {
// ============================================================================
// Metrics
// ============================================================================
-
+
public String getCoadhaleMetricsReportClassesNames() {
return getOptional("kylin.metrics.reporter-classes",
"org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter");
@@ -1629,5 +1632,4 @@ abstract public class KylinConfigBase implements Serializable {
return getOptional("kylin.tool.auto-migrate-cube.dest-config", "");
}
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c686b843/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index c01308d..f1bff41 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -83,6 +83,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
+import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Lists;
@@ -836,39 +837,41 @@ public class CubeService extends BasicService implements InitializingBean {
return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+ + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void migrateCube(CubeInstance cube, String projectName) {
KylinConfig config = cube.getConfig();
- try {
- if (!config.isAllowAutoMigrateCube()) {
- throw new InternalErrorException("This cube couldn't one click migrate cube, Please contact your ADMIN");
- }
+ if (!config.isAllowAutoMigrateCube()) {
+ throw new InternalErrorException("One click migration is disabled, please contact your ADMIN");
+ }
- for (CubeSegment segment : cube.getSegments()) {
- if (segment.getStatus() != SegmentStatusEnum.READY) {
- throw new InternalErrorException("At least one segment is not in READY state. Please check whether there are Running or Error jobs.");
- }
+ for (CubeSegment segment : cube.getSegments()) {
+ if (segment.getStatus() != SegmentStatusEnum.READY) {
+ throw new InternalErrorException(
+ "At least one segment is not in READY state. Please check whether there are Running or Error jobs.");
}
+ }
- String srcCfgUri = config.getAutoMigrateCubeSrcConfig();
- String dstCfgUri = config.getAutoMigrateCubeDestConfig();
+ String srcCfgUri = config.getAutoMigrateCubeSrcConfig();
+ String dstCfgUri = config.getAutoMigrateCubeDestConfig();
- String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true");
- String cmd = String.format(stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri, cube.getName(),
- projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge());
+ Preconditions.checkArgument(StringUtils.isNotEmpty(srcCfgUri), "Source configuration should not be empty.");
+ Preconditions.checkArgument(StringUtils.isNotEmpty(dstCfgUri),
+ "Destination configuration should not be empty.");
- logger.info("cmd: " + cmd);
+ String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true");
+ String cmd = String.format(stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri, cube.getName(),
+ projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge());
- CliCommandExecutor exec = new CliCommandExecutor();
- PatternedLogger patternedLogger = new PatternedLogger(logger);
+ logger.info("One click migration cmd: " + cmd);
- try {
- exec.execute(cmd, patternedLogger);
- } catch (IOException e) {
- throw new InternalErrorException("Migrating cube fails, Please contact your ADMIN");
- }
- } catch (Throwable e) {
- throw new InternalErrorException(e.getMessage());
+ CliCommandExecutor exec = new CliCommandExecutor();
+ PatternedLogger patternedLogger = new PatternedLogger(logger);
+
+ try {
+ exec.execute(cmd, patternedLogger);
+ } catch (IOException e) {
+ throw new InternalErrorException("Failed to perform one-click migrating", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c686b843/webapp/app/js/controllers/cubes.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubes.js b/webapp/app/js/controllers/cubes.js
index 7094ff8..a454e0a 100644
--- a/webapp/app/js/controllers/cubes.js
+++ b/webapp/app/js/controllers/cubes.js
@@ -18,7 +18,7 @@
'use strict';
-KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, cubeConfig, ProjectModel, ModelService, MetaModel, CubeList,modelsManager,TableService) {
+KylinApp.controller('CubesCtrl', function ($scope, $q, $routeParams, $location, $modal, MessageService, CubeDescService, CubeService, JobService, UserService, ProjectService, SweetAlert, loadingRequest, $log, cubeConfig, ProjectModel, ModelService, MetaModel, CubeList,modelsManager,TableService, kylinConfig) {
$scope.cubeConfig = cubeConfig;
$scope.cubeList = CubeList;