You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/23 09:30:50 UTC

[09/51] [abbrv] incubator-kylin git commit: KYLIN-544 simplified storage url by allowing shortcut like "hbase", thus deploy script gets simplified too.

KYLIN-544 simplified storage url by allowing shortcut like "hbase", thus deploy script gets simplified too.


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4f1cacb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4f1cacb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4f1cacb7

Branch: refs/heads/staging
Commit: 4f1cacb773016ecb307959f977319768e489bc97
Parents: 589e14b
Author: honma <ho...@ebay.com>
Authored: Tue Jan 20 16:03:43 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 20 16:03:43 2015 +0800

----------------------------------------------------------------------
 .../com/kylinolap/common/util/HadoopUtil.java     | 16 ++++++++--------
 examples/test_case_data/sandbox/kylin.properties  |  4 ++--
 sandbox_deploy.sh                                 | 18 ++----------------
 3 files changed, 12 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f1cacb7/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java b/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
index fc6c1a9..fbc46dd 100644
--- a/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
+++ b/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
@@ -61,6 +61,7 @@ public class HadoopUtil {
 
     /**
      * e.g.
+     * 0. hbase (recommended way)
      * 1. hbase:zk-1.hortonworks.com,zk-2.hortonworks.com,zk-3.hortonworks.com:2181:/hbase-unsecure
      * 2. hbase:zk-1.hortonworks.com,zk-2.hortonworks.com,zk-3.hortonworks.com:2181
      * 3. hbase:zk-1.hortonworks.com:2181:/hbase-unsecure
@@ -71,16 +72,16 @@ public class HadoopUtil {
         if (StringUtils.isEmpty(url))
             return conf;
 
-        // chop off "hbase:"
-        if (url.startsWith("hbase:") == false)
-            throw new IllegalArgumentException("hbase url must start with 'hbase:' -- " + url);
+        // chop off "hbase"
+        if (url.startsWith("hbase") == false)
+            throw new IllegalArgumentException("hbase url must start with 'hbase' -- " + url);
 
-        url = StringUtils.substringAfter(url, "hbase:");
+        url = StringUtils.substringAfter(url, "hbase");
         if (StringUtils.isEmpty(url))
             return conf;
 
         // case of "hbase:domain.com:2181:/hbase-unsecure"
-        Pattern urlPattern = Pattern.compile("((?:[\\w\\-.]+)(?:\\,[\\w\\-.]+)*)[:](\\d+)(?:[:](.+))?");
+        Pattern urlPattern = Pattern.compile("[:]((?:[\\w\\-.]+)(?:\\,[\\w\\-.]+)*)[:](\\d+)(?:[:](.+))");
         Matcher m = urlPattern.matcher(url);
         if (m.matches() == false)
             throw new IllegalArgumentException("HBase URL '" + url + "' is invalid, expected url is like '" + "hbase:domain.com:2181:/hbase-unsecure" + "'");
@@ -103,9 +104,8 @@ public class HadoopUtil {
         String port = m.group(2);
         conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, port);
 
-        String znodePath = m.group(3) == null ? "" : m.group(3);
-        if (StringUtils.isEmpty(znodePath) == false)
-            conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodePath);
+        String znodePath = m.group(3);
+        conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodePath);
 
         // reduce rpc retry
         conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f1cacb7/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 0675b21..5f19add 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -4,10 +4,10 @@
 kylin.rest.servers=sandbox.hortonworks.com:7070
 
 # The metadata store in hbase
-kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
+kylin.metadata.url=kylin_metadata_qa@hbase
 
 # The storage for final cube file in hbase
-kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
+kylin.storage.url=hbase
 
 # Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
 kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f1cacb7/sandbox_deploy.sh
----------------------------------------------------------------------
diff --git a/sandbox_deploy.sh b/sandbox_deploy.sh
index c5b44c5..f5b4800 100755
--- a/sandbox_deploy.sh
+++ b/sandbox_deploy.sh
@@ -113,7 +113,7 @@ source ./package.sh
 echo "retrieving classpath..."
 cd $KYLIN_HOME/job/target
 JOB_JAR_NAME="kylin-job-latest.jar"
-#generate config variables
+#export hbase configs, most of the configurations are useless now, but KYLIN_HBASE_CONF_PATH is used by SampleCubeSetupTest now
 hbase org.apache.hadoop.util.RunJar $JOB_JAR_NAME com.kylinolap.job.deployment.HbaseConfigPrinter /tmp/kylin_retrieve.sh
 #load config variables
 source /tmp/kylin_retrieve.sh
@@ -124,19 +124,13 @@ mkdir -p /etc/kylin
 HOSTNAME=`hostname`
 CLI_HOSTNAME_DEFAULT="kylin.job.remote.cli.hostname=sandbox.hortonworks.com"
 CLI_PASSWORD_DEFAULT="kylin.job.remote.cli.password=hadoop"
-METADATA_URL_DEFAULT="kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-unsecure"
-STORAGE_URL_DEFAULT="kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure"
 CHECK_URL_DEFAULT="kylin.job.yarn.app.rest.check.status.url=http://sandbox"
 
 
 NEW_CLI_HOSTNAME_PREFIX="kylin.job.remote.cli.hostname="
 NEW_CLI_PASSWORD_PREFIX="kylin.job.remote.cli.password="
-NEW_METADATA_URL_PREFIX="kylin.metadata.url=kylin_metadata_qa@hbase:"
-NEW_STORAGE_URL_PREFIX="kylin.storage.url=hbase:"
 NEW_CHECK_URL_PREFIX="kylin.job.yarn.app.rest.check.status.url=http://"
 
-KYLIN_ZOOKEEPER_URL=${KYLIN_ZOOKEEPER_QUORUM}:${KYLIN_ZOOKEEPER_CLIENT_PORT}:${KYLIN_ZOOKEEPER_ZNODE_PARENT}
-
 echo "Kylin install script requires root password for ${HOSTNAME}"
 echo "(The default root password for hortonworks VM is hadoop, and for cloudera VM is cloudera)"
 
@@ -146,18 +140,12 @@ echo "(The default root password for hortonworks VM is hadoop, and for cloudera
 CHECK_URL_DEFAULT=$(escape_sed_pattern $CHECK_URL_DEFAULT)
 CLI_HOSTNAME_DEFAULT=$(escape_sed_pattern $CLI_HOSTNAME_DEFAULT)
 CLI_PASSWORD_DEFAULT=$(escape_sed_pattern $CLI_PASSWORD_DEFAULT)
-METADATA_URL_DEFAULT=$(escape_sed_pattern $METADATA_URL_DEFAULT)
-STORAGE_URL_DEFAULT=$(escape_sed_pattern $STORAGE_URL_DEFAULT)
-
 
 NEW_CHECK_URL_PREFIX=$(escape_sed_replacement $NEW_CHECK_URL_PREFIX)
 NEW_CLI_HOSTNAME_PREFIX=$(escape_sed_replacement $NEW_CLI_HOSTNAME_PREFIX)
 NEW_CLI_PASSWORD_PREFIX=$(escape_sed_replacement $NEW_CLI_PASSWORD_PREFIX)
-NEW_METADATA_URL_PREFIX=$(escape_sed_replacement $NEW_METADATA_URL_PREFIX)
-NEW_STORAGE_URL_PREFIX=$(escape_sed_replacement $NEW_STORAGE_URL_PREFIX)
 HOSTNAME=$(escape_sed_replacement $HOSTNAME)
 ROOTPASS=$(escape_sed_replacement $ROOTPASS)
-KYLIN_ZOOKEEPER_URL=$(escape_sed_replacement $KYLIN_ZOOKEEPER_URL)
 
 
 
@@ -165,9 +153,7 @@ KYLIN_ZOOKEEPER_URL=$(escape_sed_replacement $KYLIN_ZOOKEEPER_URL)
 cat examples/test_case_data/sandbox/kylin.properties | \
     sed -e "s/${CHECK_URL_DEFAULT}/${NEW_CHECK_URL_PREFIX}${HOSTNAME}/g" | \
     sed -e "s/${CLI_HOSTNAME_DEFAULT}/${NEW_CLI_HOSTNAME_PREFIX}${HOSTNAME}/g" | \
-    sed -e "s/${CLI_PASSWORD_DEFAULT}/${NEW_CLI_PASSWORD_PREFIX}${ROOTPASS}/g" | \
-    sed -e "s/${METADATA_URL_DEFAULT}/${NEW_METADATA_URL_PREFIX}${KYLIN_ZOOKEEPER_URL}/g" | \
-    sed -e "s/${STORAGE_URL_DEFAULT}/${NEW_STORAGE_URL_PREFIX}${KYLIN_ZOOKEEPER_URL}/g" >  /etc/kylin/kylin.properties
+    sed -e "s/${CLI_PASSWORD_DEFAULT}/${NEW_CLI_PASSWORD_PREFIX}${ROOTPASS}/g" >  /etc/kylin/kylin.properties
 
 
 echo "a copy of kylin config is generated at /etc/kylin/kylin.properties:"