You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/25 07:47:24 UTC
[1/2] incubator-kylin git commit: KYLIN-1166 CubeMigrationCLI should
disable and purge the cube
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 112abda8c -> 356800307
KYLIN-1166 CubeMigrationCLI should disable and purge the cube
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/35680030
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/35680030
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/35680030
Branch: refs/heads/2.x-staging
Commit: 356800307377b8069458cdfd9169ff40aa4fcefe
Parents: 2f46301
Author: shaofengshi <sh...@apache.org>
Authored: Wed Nov 25 14:46:46 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Nov 25 14:46:58 2015 +0800
----------------------------------------------------------------------
.../storage/hbase/util/CubeMigrationCLI.java | 38 ++++++++++++++++++--
1 file changed, 36 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/35680030/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index a50b1eb..463902f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -92,7 +92,7 @@ public class CubeMigrationCLI {
private static void usage() {
System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot overwriteIfExists realExecute");
System.out.println(" srcKylinConfigUri: The KylinConfig of the cube’s source \n" + "dstKylinConfigUri: The KylinConfig of the cube’s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
- }
+ }
public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
@@ -130,6 +130,7 @@ public class CubeMigrationCLI {
if (Boolean.parseBoolean(copyAcl) == true) {
copyACL(cube);
}
+ purgeAndDisable(cubeName); // this should be the last action
if (realExecute.equalsIgnoreCase("true")) {
doOpts();
@@ -216,6 +217,11 @@ public class CubeMigrationCLI {
operations.add(new Opt(OptType.ADD_INTO_PROJECT, new Object[] { cubeName, projectName, modelName }));
}
+
+
+ private static void purgeAndDisable(String cubeName) throws IOException {
+ operations.add(new Opt(OptType.PURGE_AND_DISABLE, new Object[] { cubeName }));
+ }
private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, List<String> dictAndSnapshot) throws IOException {
@@ -240,7 +246,7 @@ public class CubeMigrationCLI {
}
private static enum OptType {
- COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, COPY_ACL
+ COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, COPY_ACL, PURGE_AND_DISABLE
}
private static class Opt {
@@ -415,6 +421,17 @@ public class CubeMigrationCLI {
}
break;
}
+
+ case PURGE_AND_DISABLE: {
+ String cubeName = (String) opt.params[0];
+ String cubeResPath = CubeInstance.concatResourcePath(cubeName);
+ Serializer<CubeInstance> cubeSerializer = new JsonSerializer<CubeInstance>(CubeInstance.class);
+ CubeInstance cube = srcStore.getResource(cubeResPath, CubeInstance.class, cubeSerializer);
+ cube.getSegments().clear();
+ cube.setStatus(RealizationStatusEnum.DISABLED);
+ srcStore.putResource(cubeResPath, cube, cubeSerializer);
+ logger.info("Cube " + cubeName + " is purged and disabled in " + srcConfig.getMetadataUrl());
+ }
}
}
@@ -455,6 +472,23 @@ public class CubeMigrationCLI {
logger.info("Undo for ADD_INTO_PROJECT is ignored");
break;
}
+ case COPY_ACL: {
+ String cubeId = (String) opt.params[0];
+ HTableInterface destAclHtable = null;
+ try {
+ destAclHtable = HBaseConnection.get(dstConfig.getMetadataUrl()).getTable(dstConfig.getMetadataUrlPrefix() + "_acl");
+
+ destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
+ destAclHtable.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(destAclHtable);
+ }
+ break;
+ }
+ case PURGE_AND_DISABLE: {
+ logger.info("Undo for PURGE_AND_DISABLE is not supported");
+ break;
+ }
}
}
}
[2/2] incubator-kylin git commit: KYLIN-1119 enhance
find-hive-dependency to support one common case
Posted by sh...@apache.org.
KYLIN-1119 enhance find-hive-dependency to support one common case
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2f46301b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2f46301b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2f46301b
Branch: refs/heads/2.x-staging
Commit: 2f46301b7be2345a296ef23cf234eaa3184154d7
Parents: 112abda
Author: shaofengshi <sh...@apache.org>
Authored: Wed Nov 25 14:39:02 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Nov 25 14:46:58 2015 +0800
----------------------------------------------------------------------
build/bin/find-hive-dependency.sh | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2f46301b/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index ee36fc3..e2c8ffa 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -44,6 +44,8 @@ then
hive_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
if [ -d "${hive_home}/hive-hcatalog" ]; then
hcatalog_home=${hive_home}/hive-hcatalog
+ elif [ -d "${hadoop_home}/hive/hcatalog" ]; then
+ hcatalog_home=${hadoop_home}/hive/hcatalog
else
echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
exit 1