[3/3] impala git commit: IMPALA-5031: Jenkins jobs should fail when UBSAN fails

[jb...@apache.org]

IMPALA-5031: Jenkins jobs should fail when UBSAN fails

Before this patch, Jenkins builds do not fail or warn if UBSAN
produces an error. This patch introduces two new failure modes,
"error" and "death".

In "error", a junit error is generated, which causes a build to be
labeled "unstable".

In "death", a forked process constantly monitors the test logs and
kills the test run if a UBSAN error is found.

Change-Id: I783243458ac2765a97a1dd7dd40d458cc2e1d80b
Reviewed-on: http://gerrit.cloudera.org:8080/11813
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins <im...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/62db9216
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/62db9216
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/62db9216

Branch: refs/heads/master
Commit: 62db9216c6ca6f7a9b4c1048fe3fb5896db8add7
Parents: 58a469c
Author: Jim Apple <jb...@apache.org>
Authored: Sat Oct 27 21:47:54 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Sun Nov 11 05:02:05 2018 +0000

----------------------------------------------------------------------
 bin/jenkins/all-tests.sh | 34 ++++++++++++++++++++++++++++++++++
 bin/jenkins/finalize.sh  |  9 +++++++++
 2 files changed, 43 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/62db9216/bin/jenkins/all-tests.sh
----------------------------------------------------------------------
diff --git a/bin/jenkins/all-tests.sh b/bin/jenkins/all-tests.sh
index d29f383..b531804 100644
--- a/bin/jenkins/all-tests.sh
+++ b/bin/jenkins/all-tests.sh
@@ -26,6 +26,40 @@ cd "${IMPALA_HOME}"
 
 export IMPALA_MAVEN_OPTIONS="-U"
 
+# When UBSAN_FAIL is "death", the logs are monitored for UBSAN errors. Any errors will
+# then cause this script to exit.
+#
+# When UBSAN_FAIL is "error", monitoring is delayed until tests have finished running.
+#
+# Any other value ignores UBSAN errors.
+: ${UBSAN_FAIL:=error}
+export UBSAN_FAIL
+
+if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \
+    && [ "${UBSAN_FAIL}" = "death" ]
+then
+  export PID_TO_KILL="$(echo $$)"
+  mkdir -p "${IMPALA_HOME}/logs"
+
+  function killer {
+    while ! grep -rI ": runtime error: " "${IMPALA_HOME}/logs"
+    do
+      sleep 1
+      if ! test -e "/proc/$PID_TO_KILL"
+      then
+        return
+      fi
+    done
+    >&2 echo "Killing process $PID_TO_KILL because it invoked undefined behavior"
+    kill -9 $PID_TO_KILL
+  }
+
+  killer &
+  export KILLER_PID="$(echo $!)"
+  disown
+  trap "kill -i $KILLER_PID" EXIT
+fi
+
 source bin/bootstrap_development.sh
 
 RET_CODE=0

http://git-wip-us.apache.org/repos/asf/impala/blob/62db9216/bin/jenkins/finalize.sh
----------------------------------------------------------------------
diff --git a/bin/jenkins/finalize.sh b/bin/jenkins/finalize.sh
index aaebb83..b7adfb4 100755
--- a/bin/jenkins/finalize.sh
+++ b/bin/jenkins/finalize.sh
@@ -21,6 +21,15 @@
 set -euo pipefail
 trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
 
+if test -v CMAKE_BUILD_TYPE && [[ "${CMAKE_BUILD_TYPE}" =~ 'UBSAN' ]] \
+    && [ "${UBSAN_FAIL}" = "error" ] \
+    && { grep -rI ": runtime error: " "${IMPALA_HOME}/logs" 2>&1 | sort | uniq \
+     | tee logs/ubsan.txt ; }
+then
+  "${IMPALA_HOME}"/bin/generate_junitxml.py --step UBSAN \
+      --stderr "${IMPALA_HOME}"/logs/ubsan.txt --error "Undefined C++ behavior"
+fi
+
 rm -rf "${IMPALA_HOME}"/logs_system
 mkdir -p "${IMPALA_HOME}"/logs_system
 dmesg > "${IMPALA_HOME}"/logs_system/dmesg