You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/12/28 04:24:10 UTC
[1/2] kylin git commit: KYLIN-1014 add reference URL and support
newer yarn version
Repository: kylin
Updated Branches:
refs/heads/1.x-staging f9de5a624 -> 2533fe5d3
KYLIN-1014 add reference URL and support newer yarn version
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2533fe5d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2533fe5d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2533fe5d
Branch: refs/heads/1.x-staging
Commit: 2533fe5d3a17dd68850a2d15bd6bcb95d3b717f2
Parents: 9d7c42b
Author: shaofengshi <sh...@apache.org>
Authored: Mon Dec 28 11:23:11 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 28 11:23:44 2015 +0800
----------------------------------------------------------------------
.../kylin/job/tools/HadoopStatusGetter.java | 30 ++++++++++++++++----
server/pom.xml | 5 +++-
2 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2533fe5d/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
index 8230d48..9035ad4 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/HadoopStatusGetter.java
@@ -140,6 +140,8 @@ public class HadoopStatusGetter {
private static String DEFAULT_KRB5_CONFIG_LOCATION = "/etc/krb5.conf";
private String getHttpResponseWithKerberosAuth(String url) throws IOException {
+
+ // referred from https://stackoverflow.com/questions/24633380/how-do-i-authenticate-with-spnego-kerberos-and-apaches-httpclient
String krb5ConfigPath = System.getProperty("java.security.krb5.conf");
if (krb5ConfigPath == null) {
krb5ConfigPath = DEFAULT_KRB5_CONFIG_LOCATION;
@@ -184,12 +186,28 @@ public class HadoopStatusGetter {
httpget.addHeader("accept", "application/json");
CloseableHttpResponse response = client.execute(httpget,context);
String redirect = null;
- org.apache.http.Header h = response.getFirstHeader("Refresh");
+ org.apache.http.Header h = response.getFirstHeader("Location");
if (h != null) {
- String s = h.getValue();
- int cut = s.indexOf("url=");
- if (cut >= 0) {
- redirect = s.substring(cut + 4);
+ redirect = h.getValue();
+ if (isValidURL(redirect) == false) {
+ log.info("Get invalid redirect url, skip it: " + redirect);
+ Thread.sleep(1000l);
+ continue;
+ }
+ } else {
+ h = response.getFirstHeader("Refresh");
+ if (h != null) {
+ String s = h.getValue();
+ int cut = s.indexOf("url=");
+ if (cut >= 0) {
+ redirect = s.substring(cut + 4);
+
+ if (isValidURL(redirect) == false) {
+ log.info("Get invalid redirect url, skip it: " + redirect);
+ Thread.sleep(1000l);
+ continue;
+ }
+ }
}
}
@@ -200,6 +218,8 @@ public class HadoopStatusGetter {
url = redirect;
log.debug("Job " + mrJobId + " check redirect url " + url + ".\n");
}
+ } catch (InterruptedException e) {
+ log.error(e.getMessage());
} finally {
httpget.releaseConnection();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2533fe5d/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index ee34cc6..d41fc07 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -176,7 +176,10 @@
<artifactId>spring-test</artifactId>
<version>${spring.framework.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </dependency>
<!-- Spring Security -->
<dependency>
<groupId>org.springframework.security</groupId>
[2/2] kylin git commit: KYLIN-1230 copy cube ACL when do release
Posted by sh...@apache.org.
KYLIN-1230 copy cube ACL when do release
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9d7c42b3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9d7c42b3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9d7c42b3
Branch: refs/heads/1.x-staging
Commit: 9d7c42b3dd9febba8960ad8dac3645e799fb0747
Parents: f9de5a6
Author: shaofengshi <sh...@apache.org>
Authored: Mon Dec 28 11:19:36 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 28 11:23:44 2015 +0800
----------------------------------------------------------------------
.../kylin/job/tools/CubeMigrationCLI.java | 63 ++++++++++++++------
1 file changed, 46 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9d7c42b3/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
index 962a4ee..44bc2c3 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
+import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.*;
import org.apache.kylin.common.util.Bytes;
@@ -72,23 +73,29 @@ public class CubeMigrationCLI {
private static FileSystem hdfsFS;
private static HBaseAdmin hbaseAdmin;
+ public static final String ACL_INFO_FAMILY = "i";
+ private static final String ACL_TABLE_NAME = "_acl";
+ private static final String ACL_INFO_FAMILY_TYPE_COLUMN = "t";
+ private static final String ACL_INFO_FAMILY_OWNER_COLUMN = "o";
+ private static final String ACL_INFO_FAMILY_PARENT_COLUMN = "p";
+
public static void main(String[] args) throws IOException, InterruptedException {
- if (args.length != 7) {
+ if (args.length != 8) {
usage();
System.exit(1);
}
- moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6]);
+ moveCube(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
}
private static void usage() {
- System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot overwriteIfExists realExecute");
- System.out.println(" srcKylinConfigUri: The KylinConfig of the cube’s source \n" + "dstKylinConfigUri: The KylinConfig of the cube’s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
+ System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName copyAclOrNot purgeOrNot overwriteIfExists realExecute");
+ System.out.println(" srcKylinConfigUri: The KylinConfig of the cube’s source \n" + "dstKylinConfigUri: The KylinConfig of the cube’s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "copyAclOrNot: true or false: whether copy cube ACL to target environment. \n" + "purgeOrNot: true or false: whether purge the cube from src server after the migration. \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
}
- public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
+ public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
srcConfig = srcCfg;
srcStore = ResourceStore.getStore(srcConfig);
@@ -122,10 +129,13 @@ public class CubeMigrationCLI {
changeHtableHost(cube);
addCubeIntoProject(cubeName, projectName);
if (Boolean.parseBoolean(copyAcl) == true) {
- copyACL(cube);
+ copyACL(cube, projectName);
}
- purgeAndDisable(cubeName); // this should be the last action
+ if (Boolean.parseBoolean(purgeAndDisable) == true) {
+ purgeAndDisable(cubeName); // this should be the last action
+ }
+
if (realExecute.equalsIgnoreCase("true")) {
doOpts();
} else {
@@ -133,9 +143,9 @@ public class CubeMigrationCLI {
}
}
- public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
+ public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
- moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, overwriteIfExists, realExecute);
+ moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute);
}
private static String checkAndGetHbaseUrl() {
@@ -178,8 +188,8 @@ public class CubeMigrationCLI {
}
}
- private static void copyACL(CubeInstance cube) {
- operations.add(new Opt(OptType.COPY_ACL, new Object[] { cube.getUuid() }));
+ private static void copyACL(CubeInstance cube, String projectName) {
+ operations.add(new Opt(OptType.COPY_ACL, new Object[] { cube.getUuid(), cube.getDescriptor().getModel().getUuid(), projectName }));
}
private static void copyFilesInMetaStore(CubeInstance cube, String overwriteIfExists) throws IOException {
@@ -402,18 +412,35 @@ public class CubeMigrationCLI {
}
case COPY_ACL: {
String cubeId = (String) opt.params[0];
+ String modelId = (String) opt.params[1];
+ String projectName = (String) opt.params[2];
+ String projectResPath = ProjectInstance.concatResourcePath(projectName);
+ Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
+ ProjectInstance project = dstStore.getResource(projectResPath, ProjectInstance.class, projectSerializer);
+ String projUUID = project.getUuid();
HTableInterface srcAclHtable = null;
HTableInterface destAclHtable = null;
try {
srcAclHtable = HBaseConnection.get(srcConfig.getMetadataUrl()).getTable(srcConfig.getMetadataUrlPrefix() + "_acl");
destAclHtable = HBaseConnection.get(dstConfig.getMetadataUrl()).getTable(dstConfig.getMetadataUrlPrefix() + "_acl");
- Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
-
- for (Cell cell : result.listCells()) {
- Put put = new Put(Bytes.toBytes(cubeId));
- put.add(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), CellUtil.cloneValue(cell));
- destAclHtable.put(put);
+ // cube acl
+ Result result = srcAclHtable.get(new Get(Bytes.toBytes(cubeId)));
+ if (result.listCells() != null) {
+ for (Cell cell : result.listCells()) {
+ byte[] family = CellUtil.cloneFamily(cell);
+ byte[] column = CellUtil.cloneQualifier(cell);
+ byte[] value = CellUtil.cloneValue(cell);
+
+ // use the target project uuid as the parent
+ if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
+ String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
+ value = Bytes.toBytes(valueString);
+ }
+ Put put = new Put(Bytes.toBytes(cubeId));
+ put.add(family, column, value);
+ destAclHtable.put(put);
+ }
}
destAclHtable.flushCommits();
} finally {
@@ -474,11 +501,13 @@ public class CubeMigrationCLI {
}
case COPY_ACL: {
String cubeId = (String) opt.params[0];
+ String modelId = (String) opt.params[1];
HTableInterface destAclHtable = null;
try {
destAclHtable = HBaseConnection.get(dstConfig.getMetadataUrl()).getTable(dstConfig.getMetadataUrlPrefix() + "_acl");
destAclHtable.delete(new Delete(Bytes.toBytes(cubeId)));
+ destAclHtable.delete(new Delete(Bytes.toBytes(modelId)));
destAclHtable.flushCommits();
} finally {
IOUtils.closeQuietly(destAclHtable);