You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/11/08 08:44:53 UTC
[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4808 Auto create
$KYLIN_HOME/hadoop_conf and link required file
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push:
new 2db2800 KYLIN-4808 Auto create $KYLIN_HOME/hadoop_conf and link required file
2db2800 is described below
commit 2db2800639f79f15d3c00774facfb31de2764e67
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Thu Nov 5 18:00:16 2020 +0800
KYLIN-4808 Auto create $KYLIN_HOME/hadoop_conf and link required file
---
build/bin/find-hadoop-conf-dir.sh | 151 ++++++++++++++++++++++++++------------
1 file changed, 104 insertions(+), 47 deletions(-)
diff --git a/build/bin/find-hadoop-conf-dir.sh b/build/bin/find-hadoop-conf-dir.sh
index ed15477..fcaa06c 100755
--- a/build/bin/find-hadoop-conf-dir.sh
+++ b/build/bin/find-hadoop-conf-dir.sh
@@ -22,59 +22,116 @@ source ${KYLIN_HOME:-"$(cd -P -- "$(dirname -- "$0")" && pwd -P)/../"}/bin/heade
echo Retrieving hadoop conf dir...
function find_hadoop_conf_dir() {
- override_hadoop_conf_dir=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.env.hadoop-conf-dir`
-
- if [ -n "$override_hadoop_conf_dir" ]; then
- verbose "kylin_hadoop_conf_dir is override as $override_hadoop_conf_dir"
- export kylin_hadoop_conf_dir=${override_hadoop_conf_dir}
+ if [[ -d ${kylin_hadoop_conf_dir} ]]; then
return
fi
+
+ override_hadoop_conf_dir=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.env.hadoop-conf-dir`
- hadoop_classpath=`hadoop classpath`
-
- arr=(`echo $hadoop_classpath | cut -d ":" -f 1- | sed 's/:/ /g'`)
- kylin_hadoop_conf_dir=
-
- for data in ${arr[@]}
- do
- result=`echo $data | grep -v -E ".*jar"`
- if [ $result ]
- then
- valid_conf_dir=true
-
- if [ ! -f $result/yarn-site.xml ]
- then
- verbose "$result is not valid hadoop dir conf because yarn-site.xml is missing"
- valid_conf_dir=false
- continue
- fi
-
- if [ ! -f $result/mapred-site.xml ]
- then
- verbose "$result is not valid hadoop dir conf because mapred-site.xml is missing"
- valid_conf_dir=false
- continue
- fi
-
- if [ ! -f $result/hdfs-site.xml ]
- then
- verbose "$result is not valid hadoop dir conf because hdfs-site.xml is missing"
- valid_conf_dir=false
- continue
- fi
-
- if [ ! -f $result/core-site.xml ]
+ if [ -n "$override_hadoop_conf_dir" ]; then
+ verbose "hadoop_conf_dir is override as $override_hadoop_conf_dir"
+ export hadoop_conf_dir=${override_hadoop_conf_dir}
+ else
+ hadoop_classpath=`hadoop classpath`
+
+ arr=(`echo $hadoop_classpath | cut -d ":" -f 1- | sed 's/:/ /g'`)
+ hadoop_conf_dir=
+
+ for data in ${arr[@]}
+ do
+ result=`echo $data | grep -v -E ".*jar"`
+ if [ $result ]
then
- verbose "$result is not valid hadoop dir conf because core-site.xml is missing"
- valid_conf_dir=false
- continue
+ valid_conf_dir=true
+
+ if [ ! -f $result/yarn-site.xml ]
+ then
+ verbose "$result is not valid hadoop dir conf because yarn-site.xml is missing"
+ valid_conf_dir=false
+ continue
+ fi
+
+ if [ ! -f $result/mapred-site.xml ]
+ then
+ verbose "$result is not valid hadoop dir conf because mapred-site.xml is missing"
+ valid_conf_dir=false
+ continue
+ fi
+
+ if [ ! -f $result/hdfs-site.xml ]
+ then
+ verbose "$result is not valid hadoop dir conf because hdfs-site.xml is missing"
+ valid_conf_dir=false
+ continue
+ fi
+
+ if [ ! -f $result/core-site.xml ]
+ then
+ verbose "$result is not valid hadoop dir conf because core-site.xml is missing"
+ valid_conf_dir=false
+ continue
+ fi
+
+ verbose "hadoop_conf_dir is $result"
+ export hadoop_conf_dir=$result
fi
-
- verbose "kylin_hadoop_conf_dir is $result"
- export kylin_hadoop_conf_dir=$result
- return
+ done
+ fi
+
+ if [ -d "$hadoop_conf_dir" ]
+ then
+ mkdir -p ${KYLIN_HOME}/hadoop_conf
+
+ checkAndLinkFile $hadoop_conf_dir/core-site.xml $kylin_hadoop_conf_dir/core-site.xml
+ checkAndLinkFile $hadoop_conf_dir/hdfs-site.xml $kylin_hadoop_conf_dir/hdfs-site.xml
+ checkAndLinkFile $hadoop_conf_dir/yarn-site.xml $kylin_hadoop_conf_dir/yarn-site.xml
+ checkAndLinkFile $hadoop_conf_dir/mapred-site.xml $kylin_hadoop_conf_dir/mapred-site.xml
+
+ checkAndLinkFile /etc/hadoop/conf/topology.py $kylin_hadoop_conf_dir/topology.py
+ checkAndLinkFile /etc/hadoop/conf/topology.map $kylin_hadoop_conf_dir/topology.map
+ checkAndLinkFile /etc/hadoop/conf/ssl-client.xml $kylin_hadoop_conf_dir/ssl-client.xml
+ checkAndLinkFile /etc/hadoop/conf/hadoop-env.sh $kylin_hadoop_conf_dir/hadoop-env.sh
+
+ copyHiveSite
fi
- done
}
+
+function checkAndLinkFile() {
+ source_file=$1
+ if [[ -f ${source_file} ]]; then
+ dst_file=$2
+ ln -s ${source_file} ${dst_file}
+ fi
+}
+
+function copyHiveSite() {
+ if [ -n "$HIVE_CONF" ]
+ then
+ if [ -f "$HIVE_CONF/hive-site.xml" ]
+ then
+ hive_site=$HIVE_CONF/hive-site.xml
+ fi
+ elif [ -n "$HIVE_HOME" ]
+ then
+ if [ -f "$HIVE_HOME/conf/hive-site.xml" ]
+ then
+ hive_site=$HIVE_HOME/conf/hive-site.xml
+ fi
+ elif [ -f /etc/hive/conf/hive-site.xml ]
+ then
+ hive_site=/etc/hive/conf/hive-site.xml
+ fi
+
+ if [ -n "$hive_site" ]
+ then
+ ln -s "${hive_site}" "${kylin_hadoop_conf_dir}/hive-site.xml"
+ else
+ echo "hive-site.xml is missing"
+ fi
+ return
+}
+
+export kylin_hadoop_conf_dir=${KYLIN_HOME}/hadoop_conf
find_hadoop_conf_dir
+
echo "export kylin_hadoop_conf_dir=$kylin_hadoop_conf_dir" > ${dir}/cached-hadoop-conf-dir.sh
\ No newline at end of file