[impala] 02/04: IMPALA-9196: Dump jstack and collect logs when tests timeout

[cs...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 482c769b0965ea25a404b9ed836890c9d66c5d94
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Sat Nov 23 18:07:51 2019 -0800

    IMPALA-9196: Dump jstack and collect logs when tests timeout
    
    This patch augments script-timeout-check.sh to also dump the jstack of
    FE when tests timeout.
    
    Tests:
     - Manually test the script with sudo privilege
     - Tested the script in private Jenkins jobs
    
    Change-Id: Ib8a5b140024c236209c7e44149660189890b9d06
    Reviewed-on: http://gerrit.cloudera.org:8080/14794
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 bin/script-timeout-check.sh | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/bin/script-timeout-check.sh b/bin/script-timeout-check.sh
index ad5fd04..4aec3af 100755
--- a/bin/script-timeout-check.sh
+++ b/bin/script-timeout-check.sh
@@ -32,6 +32,15 @@
 # pkill -P $TIMEOUT_PID || true
 # kill $TIMEOUT_PID
 
+function collect_stacktraces() {
+  name=$1
+  pid=$2
+  echo "**** Generating stacktrace of $name with process id: $pid ****"
+  gdb -ex "thread apply all bt"  --batch -p $pid > \
+      "${IMPALA_TIMEOUT_LOGS_DIR}/${name}_${pid}.txt"
+  $JAVA_HOME/bin/jstack -F $pid > "${IMPALA_TIMEOUT_LOGS_DIR}/${name}_${pid}_jstack.txt"
+}
+
 SCRIPT_NAME=""
 SLEEP_TIMEOUT_MIN=""
 
@@ -96,10 +105,16 @@ echo
 # Impala might have a thread stuck. Print the stacktrace to the console output.
 mkdir -p "$IMPALA_TIMEOUT_LOGS_DIR"
 for pid in $(pgrep impalad); do
-  echo "**** Generating stacktrace of impalad with process id: $pid ****"
-  gdb -ex "thread apply all bt"  --batch -p $pid > "${IMPALA_TIMEOUT_LOGS_DIR}/${pid}.txt"
+  collect_stacktraces impalad $pid
 done
 
+# Catalogd's process name may change. Use 'ps' directly to search the binary name.
+CATALOGD_PID=$(ps aux | grep [c]atalogd | awk '{print $2}')
+collect_stacktraces catalogd $CATALOGD_PID
+
+STATESTORED_PID=$(pgrep statestored)
+collect_stacktraces statestored $STATESTORED_PID
+
 # Now kill the caller
 kill $PPID
 
@@ -108,4 +123,3 @@ kill $PPID
 thread which can be confirmed by looking at the stacktrace of running impalad
 processes at ${IMPALA_TIMEOUT_LOGS_DIR}"
 
-