You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/01/23 09:30:50 UTC
[09/51] [abbrv] incubator-kylin git commit: KYLIN-544 simplified
storage url by allowing shortcut like "hbase",
thus deploy script gets simplified too.
KYLIN-544 simplified storage url by allowing shortcut like "hbase", thus deploy script gets simplified too.
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/4f1cacb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/4f1cacb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/4f1cacb7
Branch: refs/heads/staging
Commit: 4f1cacb773016ecb307959f977319768e489bc97
Parents: 589e14b
Author: honma <ho...@ebay.com>
Authored: Tue Jan 20 16:03:43 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 20 16:03:43 2015 +0800
----------------------------------------------------------------------
.../com/kylinolap/common/util/HadoopUtil.java | 16 ++++++++--------
examples/test_case_data/sandbox/kylin.properties | 4 ++--
sandbox_deploy.sh | 18 ++----------------
3 files changed, 12 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f1cacb7/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java b/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
index fc6c1a9..fbc46dd 100644
--- a/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
+++ b/common/src/main/java/com/kylinolap/common/util/HadoopUtil.java
@@ -61,6 +61,7 @@ public class HadoopUtil {
/**
* e.g.
+ * 0. hbase (recommended way)
* 1. hbase:zk-1.hortonworks.com,zk-2.hortonworks.com,zk-3.hortonworks.com:2181:/hbase-unsecure
* 2. hbase:zk-1.hortonworks.com,zk-2.hortonworks.com,zk-3.hortonworks.com:2181
* 3. hbase:zk-1.hortonworks.com:2181:/hbase-unsecure
@@ -71,16 +72,16 @@ public class HadoopUtil {
if (StringUtils.isEmpty(url))
return conf;
- // chop off "hbase:"
- if (url.startsWith("hbase:") == false)
- throw new IllegalArgumentException("hbase url must start with 'hbase:' -- " + url);
+ // chop off "hbase"
+ if (url.startsWith("hbase") == false)
+ throw new IllegalArgumentException("hbase url must start with 'hbase' -- " + url);
- url = StringUtils.substringAfter(url, "hbase:");
+ url = StringUtils.substringAfter(url, "hbase");
if (StringUtils.isEmpty(url))
return conf;
// case of "hbase:domain.com:2181:/hbase-unsecure"
- Pattern urlPattern = Pattern.compile("((?:[\\w\\-.]+)(?:\\,[\\w\\-.]+)*)[:](\\d+)(?:[:](.+))?");
+ Pattern urlPattern = Pattern.compile("[:]((?:[\\w\\-.]+)(?:\\,[\\w\\-.]+)*)[:](\\d+)(?:[:](.+))");
Matcher m = urlPattern.matcher(url);
if (m.matches() == false)
throw new IllegalArgumentException("HBase URL '" + url + "' is invalid, expected url is like '" + "hbase:domain.com:2181:/hbase-unsecure" + "'");
@@ -103,9 +104,8 @@ public class HadoopUtil {
String port = m.group(2);
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, port);
- String znodePath = m.group(3) == null ? "" : m.group(3);
- if (StringUtils.isEmpty(znodePath) == false)
- conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodePath);
+ String znodePath = m.group(3);
+ conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodePath);
// reduce rpc retry
conf.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f1cacb7/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index 0675b21..5f19add 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -4,10 +4,10 @@
kylin.rest.servers=sandbox.hortonworks.com:7070
# The metadata store in hbase
-kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
+kylin.metadata.url=kylin_metadata_qa@hbase
# The storage for final cube file in hbase
-kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
+kylin.storage.url=hbase
# Path to the local(relative to job engine) coprocessor jar, job engine will upload this file to hbase
kylin.coprocessor.local.jar=/tmp/kylin/kylin-storage-latest-coprocessor.jar
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/4f1cacb7/sandbox_deploy.sh
----------------------------------------------------------------------
diff --git a/sandbox_deploy.sh b/sandbox_deploy.sh
index c5b44c5..f5b4800 100755
--- a/sandbox_deploy.sh
+++ b/sandbox_deploy.sh
@@ -113,7 +113,7 @@ source ./package.sh
echo "retrieving classpath..."
cd $KYLIN_HOME/job/target
JOB_JAR_NAME="kylin-job-latest.jar"
-#generate config variables
+#export hbase configs, most of the configurations are useless now, but KYLIN_HBASE_CONF_PATH is used by SampleCubeSetupTest now
hbase org.apache.hadoop.util.RunJar $JOB_JAR_NAME com.kylinolap.job.deployment.HbaseConfigPrinter /tmp/kylin_retrieve.sh
#load config variables
source /tmp/kylin_retrieve.sh
@@ -124,19 +124,13 @@ mkdir -p /etc/kylin
HOSTNAME=`hostname`
CLI_HOSTNAME_DEFAULT="kylin.job.remote.cli.hostname=sandbox.hortonworks.com"
CLI_PASSWORD_DEFAULT="kylin.job.remote.cli.password=hadoop"
-METADATA_URL_DEFAULT="kylin.metadata.url=kylin_metadata_qa@hbase:sandbox.hortonworks.com:2181:/hbase-unsecure"
-STORAGE_URL_DEFAULT="kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure"
CHECK_URL_DEFAULT="kylin.job.yarn.app.rest.check.status.url=http://sandbox"
NEW_CLI_HOSTNAME_PREFIX="kylin.job.remote.cli.hostname="
NEW_CLI_PASSWORD_PREFIX="kylin.job.remote.cli.password="
-NEW_METADATA_URL_PREFIX="kylin.metadata.url=kylin_metadata_qa@hbase:"
-NEW_STORAGE_URL_PREFIX="kylin.storage.url=hbase:"
NEW_CHECK_URL_PREFIX="kylin.job.yarn.app.rest.check.status.url=http://"
-KYLIN_ZOOKEEPER_URL=${KYLIN_ZOOKEEPER_QUORUM}:${KYLIN_ZOOKEEPER_CLIENT_PORT}:${KYLIN_ZOOKEEPER_ZNODE_PARENT}
-
echo "Kylin install script requires root password for ${HOSTNAME}"
echo "(The default root password for hortonworks VM is hadoop, and for cloudera VM is cloudera)"
@@ -146,18 +140,12 @@ echo "(The default root password for hortonworks VM is hadoop, and for cloudera
CHECK_URL_DEFAULT=$(escape_sed_pattern $CHECK_URL_DEFAULT)
CLI_HOSTNAME_DEFAULT=$(escape_sed_pattern $CLI_HOSTNAME_DEFAULT)
CLI_PASSWORD_DEFAULT=$(escape_sed_pattern $CLI_PASSWORD_DEFAULT)
-METADATA_URL_DEFAULT=$(escape_sed_pattern $METADATA_URL_DEFAULT)
-STORAGE_URL_DEFAULT=$(escape_sed_pattern $STORAGE_URL_DEFAULT)
-
NEW_CHECK_URL_PREFIX=$(escape_sed_replacement $NEW_CHECK_URL_PREFIX)
NEW_CLI_HOSTNAME_PREFIX=$(escape_sed_replacement $NEW_CLI_HOSTNAME_PREFIX)
NEW_CLI_PASSWORD_PREFIX=$(escape_sed_replacement $NEW_CLI_PASSWORD_PREFIX)
-NEW_METADATA_URL_PREFIX=$(escape_sed_replacement $NEW_METADATA_URL_PREFIX)
-NEW_STORAGE_URL_PREFIX=$(escape_sed_replacement $NEW_STORAGE_URL_PREFIX)
HOSTNAME=$(escape_sed_replacement $HOSTNAME)
ROOTPASS=$(escape_sed_replacement $ROOTPASS)
-KYLIN_ZOOKEEPER_URL=$(escape_sed_replacement $KYLIN_ZOOKEEPER_URL)
@@ -165,9 +153,7 @@ KYLIN_ZOOKEEPER_URL=$(escape_sed_replacement $KYLIN_ZOOKEEPER_URL)
cat examples/test_case_data/sandbox/kylin.properties | \
sed -e "s/${CHECK_URL_DEFAULT}/${NEW_CHECK_URL_PREFIX}${HOSTNAME}/g" | \
sed -e "s/${CLI_HOSTNAME_DEFAULT}/${NEW_CLI_HOSTNAME_PREFIX}${HOSTNAME}/g" | \
- sed -e "s/${CLI_PASSWORD_DEFAULT}/${NEW_CLI_PASSWORD_PREFIX}${ROOTPASS}/g" | \
- sed -e "s/${METADATA_URL_DEFAULT}/${NEW_METADATA_URL_PREFIX}${KYLIN_ZOOKEEPER_URL}/g" | \
- sed -e "s/${STORAGE_URL_DEFAULT}/${NEW_STORAGE_URL_PREFIX}${KYLIN_ZOOKEEPER_URL}/g" > /etc/kylin/kylin.properties
+ sed -e "s/${CLI_PASSWORD_DEFAULT}/${NEW_CLI_PASSWORD_PREFIX}${ROOTPASS}/g" > /etc/kylin/kylin.properties
echo "a copy of kylin config is generated at /etc/kylin/kylin.properties:"