You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/05/29 07:08:33 UTC
[1/2] incubator-kylin git commit: KYLIN-753 add shell to find
hbase-common.jar in kylin.sh and then add the dependency in MR job
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7.1-staging 328cc69ff -> 07abda64a
KYLIN-753 add shell to find hbase-common.jar in kylin.sh and then add the dependency in MR job
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/7417cf13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/7417cf13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/7417cf13
Branch: refs/heads/0.7.1-staging
Commit: 7417cf131945f6af42c48625d1c38cd7cdaa7806
Parents: f418cdf
Author: shaofengshi <sh...@apache.org>
Authored: Fri May 29 11:32:47 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri May 29 12:54:52 2015 +0800
----------------------------------------------------------------------
bin/find-hbase-dependency.sh | 23 ++++++++++++++
bin/kylin.sh | 4 ++-
examples/test_case_data/sandbox/mapred-site.xml | 2 +-
.../kylin/job/hadoop/AbstractHadoopJob.java | 32 +++++++++++++-------
4 files changed, 48 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7417cf13/bin/find-hbase-dependency.sh
----------------------------------------------------------------------
diff --git a/bin/find-hbase-dependency.sh b/bin/find-hbase-dependency.sh
new file mode 100644
index 0000000..daa3db3
--- /dev/null
+++ b/bin/find-hbase-dependency.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+hbase_classpath=`hbase classpath`
+arr=(`echo $hbase_classpath | cut -d ":" --output-delimiter=" " -f 1-`)
+hbase_common_path=
+for data in ${arr[@]}
+do
+ result=`echo $data | grep -e 'hbase-common[0-9\.-]*jar'`
+ if [ $result ]
+ then
+ hbase_common_path=$data
+ fi
+done
+
+if [ -z "$hbase_common_path" ]
+then
+ echo "hbase-common lib not found"
+ exit 1
+fi
+
+hbase_dependency=${hbase_common_path}
+echo "hbase dependency: $hbase_dependency"
+export hbase_dependency
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7417cf13/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/bin/kylin.sh b/bin/kylin.sh
index 6114542..46c640f 100644
--- a/bin/kylin.sh
+++ b/bin/kylin.sh
@@ -42,8 +42,9 @@ then
then spring_profile="sandbox"
fi
- #retrive $hive_dependency
+ #retrive $hive_dependency and $hbase_dependency
source ${dir}/find-hive-dependency.sh
+ source ${dir}/find-hbase-dependency.sh
#retrive $KYLIN_EXTRA_START_OPTS
if [ -f "${dir}/setenv.sh" ]
then source ${dir}/setenv.sh
@@ -63,6 +64,7 @@ then
-Dcatalina.home=${tomcat_root} \
-Djava.io.tmpdir=${tomcat_root}/temp \
-Dkylin.hive.dependency=${hive_dependency} \
+ -Dkylin.hbase.dependency=${hbase_dependency} \
-Dspring.profiles.active=${spring_profile} \
org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start > ${tomcat_root}/logs/kylin.log 2>&1 & echo $! > ${KYLIN_HOME}/pid &
echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7417cf13/examples/test_case_data/sandbox/mapred-site.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/mapred-site.xml b/examples/test_case_data/sandbox/mapred-site.xml
index 8d345d1..f2b49c5 100644
--- a/examples/test_case_data/sandbox/mapred-site.xml
+++ b/examples/test_case_data/sandbox/mapred-site.xml
@@ -43,7 +43,7 @@
<property>
<name>mapreduce.application.classpath</name>
- <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
+ <value>/tmp/kylin/*,$HADOOP_CONF_DIR,/usr/hdp/${hdp.version}/hbase/lib/hbase-common.jar,/usr/hdp/current/hive-client/conf/,/usr/hdp/${hdp.version}/hive/lib/hive-metastore.jar,/usr/hdp/${hdp.version}/hive/lib/hive-exec.jar,/usr/hdp/${hdp.version}/hive-hcatalog/share/hcatalog/*,$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/7417cf13/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
index 9cd01a4..d95d9a9 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/AbstractHadoopJob.java
@@ -152,21 +152,31 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
String kylinHiveDependency = System.getProperty("kylin.hive.dependency");
- logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " to " + MAP_REDUCE_CLASSPATH);
+ String kylinHBaseDependency = System.getProperty("kylin.hbase.dependency");
+ logger.info("append kylin.hive.dependency: " + kylinHiveDependency + " and kylin.hive.dependency: " + kylinHBaseDependency + " to " + MAP_REDUCE_CLASSPATH);
+
+ Configuration jobConf = job.getConfiguration();
+ String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
+ if (classpath == null || classpath.length() == 0) {
+ logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value.");
+ classpath = getDefaultMapRedClasspath();
+ logger.info("The default mapred classpath is: " + classpath);
+ }
+
+
+ if (kylinHBaseDependency != null) {
+ // yarn classpath is comma separated
+ kylinHBaseDependency = kylinHBaseDependency.replace(":", ",");
+ classpath = classpath + "," + kylinHBaseDependency;
+ }
+
if (kylinHiveDependency != null) {
// yarn classpath is comma separated
kylinHiveDependency = kylinHiveDependency.replace(":", ",");
- Configuration jobConf = job.getConfiguration();
- String classpath = jobConf.get(MAP_REDUCE_CLASSPATH);
- if (classpath == null || classpath.length() == 0) {
- logger.info("Didn't find " + MAP_REDUCE_CLASSPATH + " in job configuration, will run 'mapred classpath' to get the default value.");
- classpath = getDefaultMapRedClasspath();
- logger.info("The default mapred classpath is: " + classpath);
- }
-
- jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
-
+ classpath = classpath + "," + kylinHiveDependency;
}
+
+ jobConf.set(MAP_REDUCE_CLASSPATH, classpath + "," + kylinHiveDependency);
logger.info("Hadoop job classpath is: " + job.getConfiguration().get(MAP_REDUCE_CLASSPATH));
}
[2/2] incubator-kylin git commit: Merge branch '0.7.1-staging' of
https://git-wip-us.apache.org/repos/asf/incubator-kylin into 0.7.1-staging
Posted by sh...@apache.org.
Merge branch '0.7.1-staging' of https://git-wip-us.apache.org/repos/asf/incubator-kylin into 0.7.1-staging
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/07abda64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/07abda64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/07abda64
Branch: refs/heads/0.7.1-staging
Commit: 07abda64ad5d9bcd4cb7e52ccd805364f26184e0
Parents: 7417cf1 328cc69
Author: shaofengshi <sh...@apache.org>
Authored: Fri May 29 12:56:32 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri May 29 12:56:32 2015 +0800
----------------------------------------------------------------------
docs/website/_includes/navside.html | 164 +++++++++++--------
.../2014-12-31-Build-Cube-with-Restful-API.md | 2 -
.../API/2015-03-18-Restful-Service-Doc.md | 2 -
.../2014-10-23-Kylin-web-front-end-summary.md | 45 +++++
.../2014-12-09-New-meta-data-model-structure.md | 29 ++++
...n-Kylin-test-case-with-HBase-Mini-Cluster.md | 21 +++
.../2015-03-05-Kylin-Metadata-Store.md | 45 +++++
...05-18-FAQ-on-Kylin-Installation-and-Usage.md | 46 ------
.../_posts/Getting Started/2015-05-18-FAQ.md | 44 +++++
.../Getting Started/2015-05-28-Terminology.md | 6 +
.../What-should-I-use-Kylin-for.md | 55 +++++++
.../2014-10-23-Kylin-web-front-end-summary.md | 47 ------
.../How to/2014-10-24-How-to-Contribute.md | 18 +-
...w to install kylin in your Hadoop cluster.md | 2 -
.../2014-12-09-New-meta-data-model-structure.md | 31 ----
...n-Kylin-test-case-with-HBase-Mini-Cluster.md | 23 ---
...w-to-consume-Kylin-REST-API-in-javascript.md | 2 -
.../How to/2015-03-05-Kylin-Metadata-Store.md | 47 ------
...03-18-How-to-use-kylin-remote-jdbc-driver.md | 2 -
.../2015-05-18-Export-Kylin-HBase-data.md | 2 -
...On-Hadoop-Kylin-installation-using-Docker.md | 2 -
...-22-Advance-settings-of-Kylin-environment.md | 2 -
...015-01-22-Kylin-Manual-Installation-Guide.md | 2 -
...f-Hadoop-CLI-Installation-(Dev-Env-Setup).md | 2 -
.../2015-03-13-On-Hadoop-CLI-installation.md | 2 -
.../2014-11-05-Kylin-Cube-Creation-Tutorial.md | 2 -
.../2014-11-05-Kylin-ODBC-Driver-Tutorial.md | 2 -
...in-Cube-Build-and-Job-Monitoring-Tutorial.md | 2 -
...1-11-Kylin-Cube-Permission-Grant-Tutorial.md | 2 -
.../Tutorial/2015-04-17-Kylin-Web-Tutorial.md | 2 -
.../2015-04-17-Kylin-and-Tableau-Tutorial.md | 2 -
.../blog/2015-01-25-introduce-data-model.md | 2 +-
docs/website/assets/js/main.js | 2 +-
docs/website/docs/index.html | 108 ------------
docs/website/docs/index.md | 90 ++++++++++
docs/website/docs/installation-content.html | 34 ----
docs/website/docs/intro-content.html | 72 --------
webapp/Gruntfile.js | 4 +-
38 files changed, 446 insertions(+), 521 deletions(-)
----------------------------------------------------------------------