You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/29 05:33:36 UTC
incubator-kylin git commit: KYLIN-753 add shell to find
hbase-common.jar in kylin.sh and then add the dependency in MR job
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8.0 0632016e1 -> cf25daaf9
KYLIN-753 add shell to find hbase-common.jar in kylin.sh and then add the dependency in MR job
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cf25daaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cf25daaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cf25daaf
Branch: refs/heads/0.8.0
Commit: cf25daaf989d862c94db7744c13e20dc5103aadb
Parents: 0632016
Author: shaofengshi <sh...@apache.org>
Authored: Fri May 29 11:32:47 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri May 29 11:33:05 2015 +0800
----------------------------------------------------------------------
bin/find-hbase-dependency.sh | 23 ++++++++++++++
bin/kylin.sh | 8 +++--
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
.../kylin/job/hadoop/AbstractHadoopJob.java | 32 +++++++++++++-------
4 files changed, 51 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cf25daaf/bin/find-hbase-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hbase-dependency.sh b/bin/find-hbase-dependency.sh
new file mode 100644
index 0000000..daa3db3
--- /dev/null
+++ b/bin/find-hbase-dependency.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+hbase_classpath=`hbase classpath`
+arr=(`echo $hbase_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
+hbase_common_path=
+for data in ${arr[@]}
+do
+ result=`echo $data | grep -e 'hbase-common[0-9\.-]*jar'`
+ if [ $result ]
+ then
+ hbase_common_path=$data
+ fi
+done
+
+if [ -z "$hbase_common_path" ]
+then
+ echo "hbase-common lib not found"
+ exit 1
+fi
+
+hbase_dependency=${hbase_common_path}
+echo "hbase dependency: $hbase_dependency"
+export hbase_dependency
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cf25daaf/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 9386ce8..a179fc8 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -28,8 +28,9 @@ then
then spring_profile="sandbox"
fi
- #retrive $hive_dependency
+ #retrive $hive_dependency and $hbase_dependency
source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
#retrive $KYLIN_EXTRA_START_OPTS
if [ -f "${dir}/setenv.sh" ]
then source ${dir}/setenv.sh
@@ -49,6 +50,7 @@ then
-Dcatalina.home=${tomcat_root} \
-Djava.io.tmpdir=${tomcat_root}/temp \
-Dkylin.hive.dependency=${hive_dependency} \
+ -Dkylin.hbase.dependency=${hbase_dependency} \
-Dspring.profiles.active=${spring_profile} \
org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${KYLIN_HOME}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid &
echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
@@ -92,8 +94,9 @@ then
then spring_profile="sandbox"
fi
- #retrive $hive_dependency
+ #retrive $hive_dependency and $hbase_dependency
source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
#retrive $KYLIN_EXTRA_START_OPTS
if [ -f "${dir}/setenv.sh" ]
then source ${dir}/setenv.sh
@@ -107,6 +110,7 @@ then
-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
-Dkylin.hive.dependency=${hive_dependency} \
+ -Dkylin.hbase.dependency=${hbase_dependency} \
-Dspring.profiles.active=${spring_profile} \
org.apache.kylin.job.streaming.StreamingCLI start $3 $4 > ${KYLIN_HOME}/logs/streaming_$3_$4.log 2>&1 & echo $! > ${KYLIN_HOME}/$3_$4 &
echo "streaming started $3 partition $4"
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cf25daaf/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 8d345d1..f2b49c5 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -43,7 +43,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cf25daaf/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index 1835954..60468a3 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -161,21 +161,31 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
- logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " to " + MAP_REDUCE_CLASSPATH);
+ String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency");
+ logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hive.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
+
+ Configuration jobConf = job.getConfiguration();
+ String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
+ if (classpath == null || classpath.length() == 0) {
+ logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value.");
+ classpath = getDefaultMapRedClasspath();
+ logger.info("The default mapred classpath is: " + classpath);
+ }
+
+
+ if (kylinHBaseDependency != null) {
+ // yarn classpath is comma separated
+ kylinHBaseDependency = kylinHBaseDependency.replace(":", ",");
+ classpath = classpath + "," + kylinHBaseDependency;
+ }
+
if (kylinHiveDependency != null) {
// yarn classpath is comma separated
kylinHiveDependency = kylinHiveDependency.replace(":", ",");
- Configuration jobConf = job.getConfiguration();
- String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
- if (classpath == null || classpath.length() == 0) {
- logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value.");
- classpath = getDefaultMapRedClasspath();
- logger.info("The default mapred classpath is: " + classpath);
- }
-
- jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
-
+ classpath = classpath + "," + kylinHiveDependency;
}
+
+ jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
logger.info("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
}