You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/11/07 07:59:49 UTC
[04/10] kylin git commit: KYLIN-2163 refine kylin scripts
KYLIN-2163 refine kylin scripts
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8e31ec5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8e31ec5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8e31ec5
Branch: refs/heads/yang21-cdh5.7
Commit: f8e31ec54dab0a77b3ba2885cf229a99f1c8560c
Parents: b4f9179
Author: Yang Li <li...@apache.org>
Authored: Sat Nov 5 12:00:53 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Sat Nov 5 12:00:53 2016 +0800
----------------------------------------------------------------------
build/bin/check-env.sh | 17 ++++------
build/bin/diag.sh | 7 +---
build/bin/find-hbase-dependency.sh | 7 ++--
build/bin/find-hive-dependency.sh | 31 +++++++-----------
build/bin/find-kafka-dependency.sh | 2 ++
build/bin/get-properties.sh | 2 ++
build/bin/header.sh | 41 ++++++++++++++++++++++++
build/bin/kylin.sh | 57 +++++++++++++--------------------
build/bin/metastore.sh | 7 +---
build/bin/sample.sh | 3 +-
build/bin/setenv.sh | 12 ++++---
11 files changed, 99 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/check-env.sh
----------------------------------------------------------------------
diff --git a/build/bin/check-env.sh b/build/bin/check-env.sh
index 1597c53..9cb7636 100644
--- a/build/bin/check-env.sh
+++ b/build/bin/check-env.sh
@@ -17,38 +17,33 @@
# limitations under the License.
#
-#by source
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
if [ -z "$KYLIN_HOME" ]
then
- echo 'please make sure KYLIN_HOME has been set'
- exit 1
+ quit 'Please make sure KYLIN_HOME has been set'
else
echo "KYLIN_HOME is set to ${KYLIN_HOME}"
fi
if [ -z "$(command -v hbase version)" ]
then
- echo "Please make sure the user has the privilege to run hbase shell"
- exit 1
+ quit "Please make sure the user has the privilege to run hbase shell"
fi
if [ -z "$(command -v hive --version)" ]
then
- echo "Please make sure the user has the privilege to run hive shell"
- exit 1
+ quit "Please make sure the user has the privilege to run hive shell"
fi
if [ -z "$(command -v hadoop version)" ]
then
- echo "Please make sure the user has the privilege to run hadoop shell"
- exit 1
+ quit "Please make sure the user has the privilege to run hadoop shell"
fi
WORKING_DIR=`sh $KYLIN_HOME/bin/get-properties.sh kylin.hdfs.working.dir`
hadoop fs -mkdir -p $WORKING_DIR
if [ $? != 0 ]
then
- echo "failed to create $WORKING_DIR, Please make sure the user has right to access $WORKING_DIR"
- exit 1
+ quit "Failed to create $WORKING_DIR. Please make sure the user has right to access $WORKING_DIR"
fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/diag.sh
----------------------------------------------------------------------
diff --git a/build/bin/diag.sh b/build/bin/diag.sh
index 91ba92b..5f3cec3 100644
--- a/build/bin/diag.sh
+++ b/build/bin/diag.sh
@@ -17,13 +17,8 @@
# limitations under the License.
#
-# We should set KYLIN_HOME here for multiple tomcat instances that are on the same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-KYLIN_HOME=`dirname $0`/..
-export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
-dir="$KYLIN_HOME/bin"
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-source ${dir}/check-env.sh
mkdir -p ${KYLIN_HOME}/logs
tomcat_root=${dir}/../tomcat
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-hbase-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hbase-dependency.sh b/build/bin/find-hbase-dependency.sh
index ea9a2e2..7dbb53b 100644
--- a/build/bin/find-hbase-dependency.sh
+++ b/build/bin/find-hbase-dependency.sh
@@ -17,6 +17,8 @@
# limitations under the License.
#
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
hbase_classpath=`hbase classpath`
# special handling for Amazon EMR, to prevent re-init of hbase-setenv
@@ -38,10 +40,9 @@ done
if [ -z "$hbase_common_path" ]
then
- echo "hbase-common lib not found"
- exit 1
+ quit "hbase-common lib not found"
fi
hbase_dependency=${hbase_common_path}
-echo "hbase dependency: $hbase_dependency"
+verbose "hbase dependency: $hbase_dependency"
export hbase_dependency
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-hive-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-hive-dependency.sh b/build/bin/find-hive-dependency.sh
index d17b47e..69b63a4 100644
--- a/build/bin/find-hive-dependency.sh
+++ b/build/bin/find-hive-dependency.sh
@@ -17,24 +17,18 @@
# limitations under the License.
#
-if [ -z "$KYLIN_HOME" ]
-then
- echo 'please make sure KYLIN_HOME has been set'
- exit 1
-else
- echo "KYLIN_HOME is set to ${KYLIN_HOME}"
-fi
-
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
client_mode=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.client`
hive_env=
+echo Retrieving hive dependency...
if [ "${client_mode}" == "beeline" ]
then
beeline_params=`sh ${KYLIN_HOME}/bin/get-properties.sh kylin.hive.beeline.params`
- hive_env=`beeline ${beeline_params} --outputformat=dsv -e set | grep 'env:CLASSPATH'`
+ hive_env=`beeline ${beeline_params} --outputformat=dsv -e set 2>&1 | grep 'env:CLASSPATH' `
else
- hive_env=`hive -e set | grep 'env:CLASSPATH'`
+ hive_env=`hive -e set 2>&1 | grep 'env:CLASSPATH'`
fi
hive_classpath=`echo $hive_env | grep 'env:CLASSPATH' | awk -F '=' '{print $2}'`
@@ -44,7 +38,7 @@ hive_exec_path=
if [ -n "$HIVE_CONF" ]
then
- echo "HIVE_CONF is set to: $HIVE_CONF, use it to locate hive configurations."
+ verbose "HIVE_CONF is set to: $HIVE_CONF, use it to locate hive configurations."
hive_conf_path=$HIVE_CONF
fi
@@ -69,14 +63,13 @@ done
if [ -z "$hive_conf_path" ]
then
- echo "Couldn't find hive configuration directory. Please set HIVE_CONF to the path which contains hive-site.xml."
- exit 1
+ quit "Couldn't find hive configuration directory. Please set HIVE_CONF to the path which contains hive-site.xml."
fi
# in some versions of hive hcatalog is not in hive's classpath, find it separately
if [ -z "$HCAT_HOME" ]
then
- echo "HCAT_HOME not found, try to find hcatalog path from hadoop home"
+ verbose "HCAT_HOME not found, try to find hcatalog path from hadoop home"
hadoop_home=`echo $hive_exec_path | awk -F '/hive.*/lib/' '{print $1}'`
hive_home=`echo $hive_exec_path | awk -F '/lib/' '{print $1}'`
is_aws=`uname -r | grep amzn`
@@ -90,11 +83,10 @@ then
# special handling for Amazon EMR
hcatalog_home=/usr/lib/hive-hcatalog
else
- echo "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
- exit 1
+ quit "Couldn't locate hcatalog installation, please make sure it is installed and set HCAT_HOME to the path."
fi
else
- echo "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:"
+ verbose "HCAT_HOME is set to: $HCAT_HOME, use it to find hcatalog path:"
hcatalog_home=${HCAT_HOME}
fi
@@ -102,12 +94,11 @@ hcatalog=`find -L ${hcatalog_home} -name "hive-hcatalog-core[0-9\.-]*.jar" 2>&1
if [ -z "$hcatalog" ]
then
- echo "hcatalog lib not found"
- exit 1
+ quit "hcatalog lib not found"
fi
hive_lib=`find -L "$(dirname $hive_exec_path)" -name '*.jar' ! -name '*calcite*' -printf '%p:' | sed 's/:$//'`
hive_dependency=${hive_conf_path}:${hive_lib}:${hcatalog}
-echo "hive dependency: $hive_dependency"
+verbose "hive dependency: $hive_dependency"
export hive_dependency
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/find-kafka-dependency.sh
----------------------------------------------------------------------
diff --git a/build/bin/find-kafka-dependency.sh b/build/bin/find-kafka-dependency.sh
index c6b9c24..40c8239 100644
--- a/build/bin/find-kafka-dependency.sh
+++ b/build/bin/find-kafka-dependency.sh
@@ -17,6 +17,8 @@
# limitations under the License.
#
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
kafka_home=
if [ -n "$KAFKA_HOME" ]
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index b0c7387..45bc49e 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -17,6 +17,8 @@
# limitations under the License.
#
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
+
if [ $# != 1 ]
then
echo 'invalid input'
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/header.sh
----------------------------------------------------------------------
diff --git a/build/bin/header.sh b/build/bin/header.sh
new file mode 100644
index 0000000..949ea95
--- /dev/null
+++ b/build/bin/header.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# source me
+
+if [[ "$dir" == "" ]]
+then
+ dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+
+ # set KYLIN_HOME with consideration for multiple instances that are on the same node
+ KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
+ export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
+ dir="$KYLIN_HOME/bin"
+
+ function quit {
+ echo "$@"
+ exit 1
+ }
+
+ function verbose {
+ if [[ -n "$verbose" ]]; then
+ echo "$@"
+ fi
+ }
+fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 244838b..f4a99da 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -17,12 +17,10 @@
# limitations under the License.
#
-dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
+# set verbose=true to print more logs during start up
+verbose=${verbose:-""}
-# set KYLIN_HOME with consideration for multiple instances that are on the same node
-KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
-export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
-dir="$KYLIN_HOME/bin"
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
source ${dir}/check-env.sh
mkdir -p ${KYLIN_HOME}/logs
@@ -34,8 +32,8 @@ function retrieveDependency() {
source ${dir}/find-hbase-dependency.sh
#retrive $KYLIN_EXTRA_START_OPTS
- if [ -f "${dir}/setenv.sh" ]
- then source ${dir}/setenv.sh
+ if [ -f "${dir}/setenv.sh" ]; then
+ source ${dir}/setenv.sh
fi
export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
@@ -71,10 +69,9 @@ then
spring_profile=`sh ${dir}/get-properties.sh kylin.security.profile`
if [ -z "$spring_profile" ]
then
- echo 'please set kylin.security.profile in kylin.properties, options are: testing, ldap, saml.'
- exit 1
+ quit 'please set kylin.security.profile in kylin.properties, options are: testing, ldap, saml.'
else
- echo "kylin.security.profile is set to $spring_profile"
+ verbose "kylin.security.profile is set to $spring_profile"
fi
retrieveDependency
@@ -85,15 +82,13 @@ then
if [ -z "$KYLIN_REST_ADDRESS" ]
then
kylin_rest_address=`hostname -f`":"`grep "<Connector port=" ${tomcat_root}/conf/server.xml |grep protocol=\"HTTP/1.1\" | cut -d '=' -f 2 | cut -d \" -f 2`
- echo "KYLIN_REST_ADDRESS not found, will use ${kylin_rest_address}"
else
- echo "KYLIN_REST_ADDRESS is set to: $KYLIN_REST_ADDRESS"
kylin_rest_address=$KYLIN_REST_ADDRESS
fi
+ verbose "kylin.rest.address is set to ${kylin_rest_address}"
#debug if encounter NoClassDefError
- echo "hbase classpath is:"
- hbase classpath
+ verbose "kylin classpath is: $(hbase classpath)"
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
@@ -110,9 +105,11 @@ then
-Dkylin.rest.address=${kylin_rest_address} \
-Dspring.profiles.active=${spring_profile} \
org.apache.hadoop.util.RunJar ${tomcat_root}/bin/bootstrap.jar org.apache.catalina.startup.Bootstrap start >> ${KYLIN_HOME}/logs/kylin.out 2>&1 & echo $! > ${KYLIN_HOME}/pid &
- echo "A new Kylin instance is started by $USER, stop it using \"kylin.sh stop\""
- echo "Please visit http://<ip>:7070/kylin"
- echo "You can check the log at ${KYLIN_HOME}/logs/kylin.log"
+
+ echo ""
+ echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh stop'"
+ echo "Check the log at ${KYLIN_HOME}/logs/kylin.log"
+ echo "Web UI is at http://<hostname>:7070/kylin"
exit 0
# stop command
@@ -123,18 +120,16 @@ then
PID=`cat $KYLIN_HOME/pid`
if ps -p $PID > /dev/null
then
- echo "stopping Kylin:$PID"
+ echo "Stopping Kylin: $PID"
kill $PID
rm ${KYLIN_HOME}/pid
exit 0
else
- echo "Kylin is not running, please check"
- exit 1
+ quit "Kylin is not running"
fi
else
- echo "Kylin is not running, please check"
- exit 1
+ quit "Kylin is not running"
fi
# streaming command
@@ -142,8 +137,7 @@ elif [ "$1" == "streaming" ]
then
if [ $# -lt 4 ]
then
- echo "invalid input args $@"
- exit -1
+ quit "Invalid input args $@"
fi
if [ "$2" == "start" ]
then
@@ -162,17 +156,15 @@ then
elif [ "$2" == "stop" ]
then
if [ ! -f "${KYLIN_HOME}/$3_$4" ]
- then
- echo "streaming is not running, please check"
- exit 1
+ then
+ quit "Streaming is not running"
fi
pid=`cat ${KYLIN_HOME}/$3_$4`
if [ "$pid" = "" ]
then
- echo "streaming is not running, please check"
- exit 1
+ quit "Streaming is not running"
else
- echo "stopping streaming:$pid"
+ echo "Stopping streaming: $pid"
kill $pid
fi
rm ${KYLIN_HOME}/$3_$4
@@ -184,8 +176,6 @@ then
# monitor command
elif [ "$1" == "monitor" ]
then
- echo "monitor job"
-
retrieveDependency
source ${dir}/find-kafka-dependency.sh
@@ -222,6 +212,5 @@ then
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=kylin-log4j.properties "$@"
else
- echo "usage: kylin.sh start or kylin.sh stop"
- exit 1
+ quit "Usage: 'kylin.sh start' or 'kylin.sh stop'"
fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index a7a9e27..72afbae 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -24,12 +24,7 @@
# take a look at SandboxMetastoreCLI
-dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
-
-# set KYLIN_HOME with consideration for multiple instances that are on the same node
-KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
-export KYLIN_HOME=`cd "$KYLIN_HOME"; pwd`
-dir="$KYLIN_HOME/bin"
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
source ${dir}/check-env.sh
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index fdb8601..e420df7 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -17,8 +17,7 @@
# limitations under the License.
#
-dir=$(cd -P -- "$(dirname -- "$0")" && pwd -P)
-export KYLIN_HOME=${KYLIN_HOME:-"${dir}/../"}
+source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
source ${dir}/check-env.sh
job_jar=`find -L ${KYLIN_HOME}/lib/ -name kylin-job*.jar`
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8e31ec5/build/bin/setenv.sh
----------------------------------------------------------------------
diff --git a/build/bin/setenv.sh b/build/bin/setenv.sh
index 24084f3..f6cf210 100755
--- a/build/bin/setenv.sh
+++ b/build/bin/setenv.sh
@@ -17,6 +17,8 @@
# limitations under the License.
#
+# source me
+
# (if your're deploying KYLIN on a powerful server and want to replace the default conservative settings)
# uncomment following to for it to take effect
export KYLIN_JVM_SETTINGS="-Xms1024M -Xmx4096M -Xss1024K -XX:MaxPermSize=128M -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$KYLIN_HOME/logs/kylin.gc.$$ -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=64M"
@@ -35,21 +37,21 @@ then
echo "KYLIN_JVM_SETTINGS is ${KYLIN_JVM_SETTINGS}"
KYLIN_EXTRA_START_OPTS="${KYLIN_JVM_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
else
- echo "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}"
+ verbose "KYLIN_JVM_SETTINGS is not set, using default jvm settings: ${KYLIN_JVM_SETTINGS}"
fi
if [ ! -z "${KYLIN_DEBUG_SETTINGS}" ]
then
- echo "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}"
+ verbose "KYLIN_DEBUG_SETTINGS is ${KYLIN_DEBUG_SETTINGS}"
KYLIN_EXTRA_START_OPTS="${KYLIN_DEBUG_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
else
- echo "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging"
+ verbose "KYLIN_DEBUG_SETTINGS is not set, will not enable remote debuging"
fi
if [ ! -z "${KYLIN_LD_LIBRARY_SETTINGS}" ]
then
- echo "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
+ verbose "KYLIN_LD_LIBRARY_SETTINGS is ${KYLIN_LD_LIBRARY_SETTINGS}"
KYLIN_EXTRA_START_OPTS="${KYLIN_LD_LIBRARY_SETTINGS} ${KYLIN_EXTRA_START_OPTS}"
else
- echo "KYLIN_LD_LIBRARY_SETTINGS is not set, Usually it's okay unless you want to specify your own native path"
+ verbose "KYLIN_LD_LIBRARY_SETTINGS is not set, it is okay unless you want to specify your own native path"
fi