[kylin] branch master updated: [KYLIN-4472] After running docker image, it sometimes failed to run kylin successfully.

[sh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git


The following commit(s) were added to refs/heads/master by this push:
     new 525290f  [KYLIN-4472] After running docker image, it sometimes failed to run kylin successfully.
525290f is described below

commit 525290faf6729275423a4ecfddc42386a2addbbf
Author: Zhichao Zhang <44...@qq.com>
AuthorDate: Wed Apr 29 12:44:38 2020 +0800

    [KYLIN-4472] After running docker image, it sometimes failed to run kylin successfully.
    
    Problems:
    When run docker image for the first time, sometimes hive command will be executed concurrently and it will insert two version records into the VERSION table, this will lead to start kylin unsuccessfully.
    
    error messages:
    Something wrong with Hive CLI or Beeline, please execute Hive CLI or Beeline CLI in terminal to find the root cause.
    
    Solutions:
    call script $KYLIN_HOME/bin/check-hive-usability.sh before all kylin.sh, it will insert one version record into VERSION table .
---
 docker/entrypoint.sh | 61 ++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 18 deletions(-)

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index edeefec..e207f8d 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -18,17 +18,22 @@
 #
 echo "127.0.0.1 sandbox.hortonworks.com" >> /etc/hosts
 
+# clean pid files
+rm -f /tmp/*.pid
+
 # start mysql
 service mysqld start
-mysqladmin -uroot password 123456
-mysql -uroot -p123456 -e "grant all privileges on root.* to root@'%' identified by '123456';"
+if [ ! -f "/home/admin/first_run" ]
+then
+    mysqladmin -uroot password 123456
+    mysql -uroot -p123456 -e "grant all privileges on root.* to root@'%' identified by '123456';"
+fi
 
 # start hdfs
 if [ ! -f "/home/admin/first_run" ]
 then
     hdfs namenode -format
 fi
-touch /home/admin/first_run
 $HADOOP_HOME/sbin/hadoop-daemon.sh start namenode
 $HADOOP_HOME/sbin/hadoop-daemon.sh start datanode
 
@@ -40,7 +45,10 @@ $HADOOP_HOME/sbin/yarn-daemon.sh start nodemanager
 $HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start historyserver
 
 # start hbase
-rm -rf /data/zookeeper/*
+if [ ! -f "/home/admin/first_run" ]
+then
+    rm -rf /data/zookeeper/*
+fi
 $HBASE_HOME/bin/start-hbase.sh
 
 # start kafka
@@ -48,24 +56,41 @@ rm -rf /tmp/kafka-logs
 nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties &
 
 # start livy
-hdfs dfs -mkdir -p /kylin/livy
-hdfs dfs -put -f $HBASE_HOME/lib/hbase-client-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
-hdfs dfs -put -f $HBASE_HOME/lib/hbase-common-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
-hdfs dfs -put -f $HBASE_HOME/lib/hbase-hadoop-compat-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
-hdfs dfs -put -f $HBASE_HOME/lib/hbase-hadoop2-compat-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
-hdfs dfs -put -f $HBASE_HOME/lib/hbase-server-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
-hdfs dfs -put -f $HBASE_HOME/lib/htrace-core-*-incubating.jar hdfs://localhost:9000/kylin/livy/
-hdfs dfs -put -f $HBASE_HOME/lib/metrics-core-*.jar hdfs://localhost:9000/kylin/livy/
-hdfs dfs -put -f $KYLIN_HOME/lib/kylin-job-$KYLIN_VERSION.jar hdfs://localhost:9000/kylin/livy/
+if [ ! -f "/home/admin/first_run" ]
+then
+    hdfs dfs -mkdir -p /kylin/livy
+    hdfs dfs -put -f $HBASE_HOME/lib/hbase-client-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+    hdfs dfs -put -f $HBASE_HOME/lib/hbase-common-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+    hdfs dfs -put -f $HBASE_HOME/lib/hbase-hadoop-compat-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+    hdfs dfs -put -f $HBASE_HOME/lib/hbase-hadoop2-compat-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+    hdfs dfs -put -f $HBASE_HOME/lib/hbase-server-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+    hdfs dfs -put -f $HBASE_HOME/lib/htrace-core-*-incubating.jar hdfs://localhost:9000/kylin/livy/
+    hdfs dfs -put -f $HBASE_HOME/lib/metrics-core-*.jar hdfs://localhost:9000/kylin/livy/
+    hdfs dfs -put -f $KYLIN_HOME/lib/kylin-job-$KYLIN_VERSION.jar hdfs://localhost:9000/kylin/livy/
+fi
 $LIVY_HOME/bin/livy-server start
 
+mkdir -p ${KYLIN_HOME}/logs
+# check hive usability first, this operation will insert one version record into VERSION table
+$KYLIN_HOME/bin/check-hive-usability.sh > ${KYLIN_HOME}/logs/kylin-verbose.log 2>&1
+# wait for starting hbase server successfully
+sleep 20s
+
 # prepare kafka topic and data
-$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic kylin_streaming_topic
-nohup $KYLIN_HOME/bin/kylin.sh org.apache.kylin.source.kafka.util.KafkaSampleProducer --topic kylin_streaming_topic --broker localhost:9092 < /dev/null 2>&1 > /tmp/kafka-sample.log &
-# create sample cube
-sh $KYLIN_HOME/bin/sample.sh
+if [ ! -f "/home/admin/first_run" ]
+then
+    $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic kylin_streaming_topic
+fi
+
+nohup $KYLIN_HOME/bin/kylin.sh org.apache.kylin.source.kafka.util.KafkaSampleProducer --topic kylin_streaming_topic --broker localhost:9092 > /dev/null 2>&1 > /tmp/kafka-sample.log &
+# create sample cube at the first time
+if [ ! -f "/home/admin/first_run" ]
+then
+    sh $KYLIN_HOME/bin/sample.sh >> ${KYLIN_HOME}/logs/kylin-verbose.log 2>&1
+fi
+touch /home/admin/first_run
 # start kylin
-$KYLIN_HOME/bin/kylin.sh start
+$KYLIN_HOME/bin/kylin.sh -v start >> ${KYLIN_HOME}/logs/kylin-verbose.log 2>&1
 
 while :
 do