[impala] 01/05: IMPALA-10676: Improve start/stop scripts for Hiveserver and Metastore

[st...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit d2a1460b92a20404582166c5588e9c5319b97292
Author: Vihang Karajgaonkar <vi...@cloudera.com>
AuthorDate: Thu Apr 15 15:08:16 2021 -0500

    IMPALA-10676: Improve start/stop scripts for Hiveserver and Metastore
    
    - Separate metastore and hiveserver starting in run-hive-server.sh
    - Change kill-hive-server.sh to shut down Metastore and retry
    
    Change-Id: Ie9208efdf49f383c5cfb10cd9881272847405a05
    Reviewed-on: http://gerrit.cloudera.org:8080/17340
    Reviewed-by: Joe McDonnell <jo...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
    Reviewed-by: Vihang Karajgaonkar <vi...@cloudera.com>
---
 testdata/bin/kill-hive-server.sh | 32 ++++++++++++++++++++++++++++++--
 testdata/bin/run-hive-server.sh  | 36 ++++++++++++++++++++++++++++--------
 2 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/testdata/bin/kill-hive-server.sh b/testdata/bin/kill-hive-server.sh
index 59c44e5..b79435c 100755
--- a/testdata/bin/kill-hive-server.sh
+++ b/testdata/bin/kill-hive-server.sh
@@ -22,5 +22,33 @@ set -euo pipefail
 setup_report_build_error
 
 DIR=$(dirname "$0")
-echo Stopping Hive
-"$DIR"/kill-java-service.sh -c HiveServer -c HiveMetaStore
+KILL_HIVESERVER=1
+KILL_METASTORE=1
+
+while [ -n "$*" ]
+do
+  case $1 in
+    -only_hiveserver)
+      KILL_METASTORE=0
+      ;;
+    -only_metastore)
+      KILL_HIVESERVER=0
+      ;;
+    -help|-h|*)
+      echo "kill-hive-server.sh : Kills the hive server and the metastore."
+      echo "[-only_metastore] : Only kills the hive metastore."
+      echo "[-only_hiveserver] : Only kills the hive server."
+      exit 1;
+      ;;
+    esac
+  shift;
+done
+
+if [[ $KILL_HIVESERVER -eq 1 ]]; then
+  echo Stopping Hive server.
+  "$DIR"/kill-java-service.sh -c HiveServer
+fi
+if [[ $KILL_METASTORE -eq 1 ]]; then
+  echo Stopping Hive metastore.
+  "$DIR"/kill-java-service.sh -c HiveMetaStore
+fi
diff --git a/testdata/bin/run-hive-server.sh b/testdata/bin/run-hive-server.sh
index 25986c5..f8c3ce5 100755
--- a/testdata/bin/run-hive-server.sh
+++ b/testdata/bin/run-hive-server.sh
@@ -27,7 +27,8 @@ HIVE_METASTORE_PORT=9083
 LOGDIR=${IMPALA_CLUSTER_LOGS_DIR}/hive
 HIVES2_TRANSPORT="plain_sasl"
 METASTORE_TRANSPORT="buffered"
-ONLY_METASTORE=0
+START_METASTORE=1
+START_HIVESERVER=1
 ENABLE_RANGER_AUTH=0
 
 CLUSTER_BIN=${IMPALA_HOME}/testdata/bin
@@ -47,15 +48,19 @@ while [ -n "$*" ]
 do
   case $1 in
     -only_metastore)
-      ONLY_METASTORE=1
+      START_HIVESERVER=0
       ;;
     -with_ranger)
       ENABLE_RANGER_AUTH=1
       echo "Starting Hive with Ranger authorization."
       ;;
+    -only_hiveserver)
+      START_METASTORE=0
+      ;;
     -help|-h|*)
       echo "run-hive-server.sh : Starts the hive server and the metastore."
       echo "[-only_metastore] : Only starts the hive metastore."
+      echo "[-only_hiveserver] : Only starts the hive server."
       echo "[-with_ranger] : Starts with Ranger authorization (only for Hive 3)."
       exit 1;
       ;;
@@ -63,9 +68,20 @@ do
   shift;
 done
 
+if [[ $START_METASTORE -eq 0 && $START_HIVESERVER -eq 0 ]]; then
+  echo "Skipping metastore and hiveserver. Nothing to do"
+  exit 1;
+fi
+
 # TODO: We should have a retry loop for every service we start.
 # Kill for a clean start.
-${CLUSTER_BIN}/kill-hive-server.sh &> /dev/null
+if [[ $START_HIVESERVER -eq 1 ]]; then
+  ${CLUSTER_BIN}/kill-hive-server.sh -only_hiveserver &> /dev/null
+fi
+
+if [[ $START_METASTORE -eq 1 ]]; then
+  ${CLUSTER_BIN}/kill-hive-server.sh -only_metastore &> /dev/null
+fi
 
 export HIVE_METASTORE_HADOOP_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,\
 suspend=n,address=30010"
@@ -114,16 +130,20 @@ export KUDU_SKIP_HMS_PLUGIN_VALIDATION=${KUDU_SKIP_HMS_PLUGIN_VALIDATION:-1}
 # Starts a Hive Metastore Server on the specified port.
 # To debug log4j2 loading issues, add to HADOOP_CLIENT_OPTS:
 #   -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=TRACE
-HADOOP_CLIENT_OPTS="-Xmx2024m -Dhive.log.file=hive-metastore.log" hive \
-  --service metastore -p $HIVE_METASTORE_PORT > ${LOGDIR}/hive-metastore.out 2>&1 &
+if [ ${START_METASTORE} -eq 1 ]; then
+  HADOOP_CLIENT_OPTS="-Xmx2024m -Dhive.log.file=hive-metastore.log" hive \
+      --service metastore -p $HIVE_METASTORE_PORT > ${LOGDIR}/hive-metastore.out 2>&1 &
 
-# Wait for the Metastore to come up because HiveServer2 relies on it being live.
-${CLUSTER_BIN}/wait-for-metastore.py --transport=${METASTORE_TRANSPORT}
+  # Wait for the Metastore to come up because HiveServer2 relies on it being live.
+  ${CLUSTER_BIN}/wait-for-metastore.py --transport=${METASTORE_TRANSPORT}
+fi
 
 # Include the latest libfesupport.so in the JAVA_LIBRARY_PATH
 export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH-}:${IMPALA_HOME}/be/build/latest/service/"
 
-if [ ${ONLY_METASTORE} -eq 0 ]; then
+export HIVESERVER2_HADOOP_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,\
+suspend=n,address=30020"
+if [ ${START_HIVESERVER} -eq 1 ]; then
   # Starts a HiveServer2 instance on the port specified by the HIVE_SERVER2_THRIFT_PORT
   # environment variable. HADOOP_HEAPSIZE should be set to at least 2048 to avoid OOM
   # when loading ORC tables like widerow.