You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/11/24 06:41:53 UTC
[1/2] incubator-kylin git commit: KYLIN-1157 copy ACL when migrate
cube
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging eb568c02e -> dff3c03f6
KYLIN-1157 copy ACL when migrate cube
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dff3c03f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dff3c03f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dff3c03f
Branch: refs/heads/2.x-staging
Commit: dff3c03f6065727d95cc8b90a13f9c0fbe125f46
Parents: 2178fc1
Author: shaofengshi <sh...@apache.org>
Authored: Tue Nov 24 11:46:36 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Nov 24 13:41:46 2015 +0800
----------------------------------------------------------------------
.../storage/hbase/util/CubeMigrationCLI.java | 67 ++++++++++++++------
1 file changed, 48 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dff3c03f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 6a4d5e3..a50b1eb 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -18,24 +18,20 @@
package org.apache.kylin.storage.hbase.util;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.*;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -54,10 +50,14 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
/**
* <p/>
@@ -81,20 +81,20 @@ public class CubeMigrationCLI {
public static void main(String[] args) throws IOException, InterruptedException {
- if (args.length != 6) {
+ if (args.length != 7) {
usage();
System.exit(1);
}
- moveCube(args[0], args[1], args[2], args[3], args[4], args[5]);
+ moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6]);
}
private static void usage() {
- System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName overwriteIfExists realExecute");
- System.out.println(" srcKylinConfigUri: The KylinConfig of the cube’s source \n" + "dstKylinConfigUri: The KylinConfig of the cube’s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
- }
+ System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot overwriteIfExists realExecute");
+ System.out.println(" srcKylinConfigUri: The KylinConfig of the cube’s source \n" + "dstKylinConfigUri: The KylinConfig of the cube’s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
+ }
- public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
+ public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
srcConfig = srcCfg;
srcStore = ResourceStore.getStore(srcConfig);
@@ -127,6 +127,9 @@ public class CubeMigrationCLI {
renameFoldersInHdfs(cube);
changeHtableHost(cube);
addCubeIntoProject(cubeName, projectName, cube.getModelName());
+ if (Boolean.parseBoolean(copyAcl) == true) {
+ copyACL(cube);
+ }
if (realExecute.equalsIgnoreCase("true")) {
doOpts();
@@ -135,8 +138,8 @@ public class CubeMigrationCLI {
}
}
- public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
- moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, overwriteIfExists, realExecute);
+ public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
+ moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, overwriteIfExists, realExecute);
}
private static String checkAndGetHbaseUrl() {
@@ -184,6 +187,10 @@ public class CubeMigrationCLI {
}
}
+ private static void copyACL(CubeInstance cube) {
+ operations.add(new Opt(OptType.COPY_ACL, new Object[] { cube.getUuid() }));
+ }
+
private static void copyFilesInMetaStore(CubeInstance cube, String overwriteIfExists) throws IOException {
List<String> metaItems = new ArrayList<String>();
@@ -233,7 +240,7 @@ public class CubeMigrationCLI {
}
private static enum OptType {
- COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST
+ COPY_FILE_IN_META, COPY_DICT_OR_SNAPSHOT, RENAME_FOLDER_IN_HDFS, ADD_INTO_PROJECT, CHANGE_HTABLE_HOST, COPY_ACL
}
private static class Opt {
@@ -386,6 +393,28 @@ public class CubeMigrationCLI {
logger.info("Project instance for " + projectName + " is corrected");
break;
}
+ case COPY_ACL: {
+ String cubeId = (String) opt.params[0];
+ HTableInterface srcAclHtable = null;
+ HTableInterface destAclHtable = null;
+ try {
+ srcAclHtable = HBaseConnection.get(srcConfig.getMetadataUrl()).getTable(srcConfig.getMetadataUrlPrefix() + "_acl");
+ destAclHtable = HBaseConnection.get(dstConfig.getMetadataUrl()).getTable(dstConfig.getMetadataUrlPrefix() + "_acl");
+
+ Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
+
+ for (Cell cell : result.listCells()) {
+ Put put = new Put(Bytes.toBytes(cubeId));
+ put.add(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), CellUtil.cloneValue(cell));
+ destAclHtable.put(put);
+ }
+ destAclHtable.flushCommits();
+ } finally {
+ IOUtils.closeQuietly(srcAclHtable);
+ IOUtils.closeQuietly(destAclHtable);
+ }
+ break;
+ }
}
}
[2/2] incubator-kylin git commit: KYLIN-1162 be compatible with
YARN-2605
Posted by sh...@apache.org.
KYLIN-1162 be compatible with YARN-2605
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/2178fc16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/2178fc16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/2178fc16
Branch: refs/heads/2.x-staging
Commit: 2178fc1661db469f7e9f7e9f9e07b25a4a5594fe
Parents: eb568c0
Author: shaofengshi <sh...@apache.org>
Authored: Tue Nov 24 11:41:27 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Tue Nov 24 13:41:46 2015 +0800
----------------------------------------------------------------------
.../engine/mr/common/HadoopStatusGetter.java | 32 ++++++++++++++------
1 file changed, 23 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/2178fc16/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
index 7bab771..fd7afd3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopStatusGetter.java
@@ -79,16 +79,27 @@ public class HadoopStatusGetter {
client.executeMethod(get);
String redirect = null;
- Header h = get.getResponseHeader("Refresh");
+ Header h = get.getResponseHeader("Location");
if (h != null) {
- String s = h.getValue();
- int cut = s.indexOf("url=");
- if (cut >= 0) {
- redirect = s.substring(cut + 4);
-
- if (isValidURL(redirect) == false) {
- logger.info("Get invalid redirect url, skip it: " + redirect);
- continue;
+ redirect = h.getValue();
+ if (isValidURL(redirect) == false) {
+ logger.info("Get invalid redirect url, skip it: " + redirect);
+ Thread.sleep(1000l);
+ continue;
+ }
+ } else {
+ h = get.getResponseHeader("Refresh");
+ if (h != null) {
+ String s = h.getValue();
+ int cut = s.indexOf("url=");
+ if (cut >= 0) {
+ redirect = s.substring(cut + 4);
+
+ if (isValidURL(redirect) == false) {
+ logger.info("Get invalid redirect url, skip it: " + redirect);
+ Thread.sleep(1000l);
+ continue;
+ }
}
}
}
@@ -100,6 +111,8 @@ public class HadoopStatusGetter {
url = redirect;
logger.debug("Job " + mrJobId + " check redirect url " + url + ".\n");
}
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage());
} finally {
get.releaseConnection();
}
@@ -132,4 +145,5 @@ public class HadoopStatusGetter {
return false;
}
+
}