You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/06/24 07:20:14 UTC
kylin git commit: KYLIN-1818: change kafka dep to provided
Repository: kylin
Updated Branches:
refs/heads/master 4ffe818e2 -> 25081d928
KYLIN-1818: change kafka dep to provided
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/25081d92
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/25081d92
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/25081d92
Branch: refs/heads/master
Commit: 25081d928ef081c343044af3587dc7e2830a4a46
Parents: 4ffe818
Author: Yiming Liu <li...@gmail.com>
Authored: Fri Jun 24 15:13:14 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jun 24 15:17:33 2016 +0800
----------------------------------------------------------------------
assembly/pom.xml | 1 -
build/bin/find-kafka-dependency.sh | 51 +++++++++++++++++++++++++++++++++
build/bin/kylin.sh | 4 +++
3 files changed, 55 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/25081d92/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index d643b62..d7764a0 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -177,7 +177,6 @@
<artifactId>kafka_2.10</artifactId>
<version>${kafka.version}</version>
<scope>provided</scope>
- <!-- FIXME: Should be provided just like hive and hbase, inflates job jar from 9 MB to 21 MB -->
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/25081d92/build/bin/find-kafka-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-kafka-dependency.sh b/build/bin/find-kafka-dependency.sh
new file mode 100644
index 0000000..c6b9c24
--- /dev/null
+++ b/build/bin/find-kafka-dependency.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+kafka_home=
+
+if [ -n "$KAFKA_HOME" ]
+then
+ echo "KAFKA_HOME is set to: $KAFKA_HOME, use it to locate kafka dependencies."
+ kafka_home=$KAFKA_HOME
+fi
+
+if [ -z "$KAFKA_HOME" ]
+then
+ echo "Couldn't find kafka home. Please set KAFKA_HOME to the path which contains kafka dependencies."
+ exit 1
+fi
+
+# works for kafka 9+
+kafka_client=`find -L "$(dirname $kafka_home)" -name 'kafka-clients-[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
+if [ -z "$kafka_client" ]
+then
+# works for kafka 8
+ kafka_broker=`find -L "$(dirname $kafka_home)" -name 'kafka_[a-z0-9A-Z\.-]*.jar' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
+ if [ -z "$kafka_broker" ]
+ then
+ echo "kafka client lib not found"
+ exit 1
+ else
+ echo "kafka dependency: $kafka_broker"
+ export kafka_dependency
+ fi
+else
+ echo "kafka dependency: $kafka_client"
+ export kafka_dependency
+fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/25081d92/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 2ea3a8c..f3710c3 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -141,6 +141,7 @@ then
#retrive $hive_dependency and $hbase_dependency
source ${dir}/find-hive-dependency.sh
source ${dir}/find-hbase-dependency.sh
+ source ${dir}/find-kafka-dependency.sh
#retrive $KYLIN_EXTRA_START_OPTS
if [ -f "${dir}/setenv.sh" ]
then source ${dir}/setenv.sh
@@ -153,6 +154,7 @@ then
hbase ${KYLIN_EXTRA_START_OPTS} \
-Dlog4j.configuration=kylin-log4j.properties\
-Dkylin.hive.dependency=${hive_dependency} \
+ -Dkylin.kafka.dependency=${kafka_dependency} \
-Dkylin.hbase.dependency=${hbase_dependency} \
org.apache.kylin.engine.streaming.cli.StreamingCLI $@ > ${KYLIN_HOME}/logs/streaming_$3_$4.log 2>&1 & echo $! > ${KYLIN_HOME}/logs/$3_$4 &
echo "streaming started name: $3 id: $4"
@@ -187,6 +189,7 @@ then
#retrive $hive_dependency and $hbase_dependency
source ${dir}/find-hive-dependency.sh
source ${dir}/find-hbase-dependency.sh
+ source ${dir}/find-kafka-dependency.sh
#retrive $KYLIN_EXTRA_START_OPTS
if [ -f "${dir}/setenv.sh" ]
then source ${dir}/setenv.sh
@@ -199,6 +202,7 @@ then
hbase ${KYLIN_EXTRA_START_OPTS} \
-Dlog4j.configuration=kylin-log4j.properties\
-Dkylin.hive.dependency=${hive_dependency} \
+ -Dkyiln.kafka.dependency=${kafka_dependency} \
-Dkylin.hbase.dependency=${hbase_dependency} \
org.apache.kylin.engine.streaming.cli.MonitorCLI $@ > ${KYLIN_HOME}/logs/monitor.log 2>&1
exit 0