You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ph...@apache.org on 2018/05/22 03:33:57 UTC
[2/4] impala git commit: Moving default sanitizer options into
init.cc from shell scripts.
Moving default sanitizer options into init.cc from shell scripts.
When running tests with ASAN, you need to set ASAN_OPTIONS explicitly,
to avoid various failures. In particular, backend tests fail complaining
about memory leaks and tests that use the parquet-reader binary complain
similarly. It turns out that we can shove the default options into our
code base directly, avoiding the need for users to set it explicitly.
I've done the same thing for TSAN and UBSAN.
I've manually checked that these are being read. In the UBSAN case, I
checked both with gdb and with inotifywatch on the suppressions file.
Change-Id: I3cbbd210c67750a48003f336bea1f3e1cb2d9e6b
Reviewed-on: http://gerrit.cloudera.org:8080/10404
Reviewed-by: Jim Apple <jb...@apache.org>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/2caec900
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/2caec900
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/2caec900
Branch: refs/heads/2.x
Commit: 2caec900502e9991f53154268b6d2b52f5c78ea2
Parents: 69e88f7
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Mon May 14 20:36:57 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Mon May 21 20:10:17 2018 +0000
----------------------------------------------------------------------
be/CMakeLists.txt | 2 +-
be/src/common/init.cc | 26 ++++++++++++++++++++++++++
bin/run-backend-tests.sh | 4 +---
bin/start-catalogd.sh | 4 ----
bin/start-impalad.sh | 4 ----
bin/start-statestored.sh | 4 ----
6 files changed, 28 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/2caec900/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index cc5a597..2ef7b85 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -112,7 +112,7 @@ SET(CXX_FLAGS_ADDRESS_SANITIZER
# Set the flags to the undefined behavior sanitizer, also known as "ubsan"
# Turn on sanitizer and debug symbols to get stack traces:
-SET(CXX_FLAGS_UBSAN "${CXX_CLANG_FLAGS} -ggdb3 -fno-omit-frame-pointer -fsanitize=undefined")
+SET(CXX_FLAGS_UBSAN "${CXX_CLANG_FLAGS} -ggdb3 -fno-omit-frame-pointer -fsanitize=undefined -DUNDEFINED_SANITIZER")
# Add flags to enable symbol resolution in the stack traces:
SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -rtlib=compiler-rt -lgcc_s")
# Ignore a number of noisy errors with too many false positives:
http://git-wip-us.apache.org/repos/asf/impala/blob/2caec900/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 745112a..9daaf5c 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -274,3 +274,29 @@ Status impala::StartMemoryMaintenanceThread() {
return Thread::Create("common", "memory-maintenance-thread",
&MemoryMaintenanceThread, &memory_maintenance_thread);
}
+
+#if defined(ADDRESS_SANITIZER)
+// Default ASAN_OPTIONS. Override by setting environment variable $ASAN_OPTIONS.
+extern "C" const char *__asan_default_options() {
+ // IMPALA-2746: backend tests don't pass with leak sanitizer enabled.
+ return "handle_segv=0 detect_leaks=0 allocator_may_return_null=1";
+}
+#endif
+
+#if defined(THREAD_SANITIZER)
+// Default TSAN_OPTIONS. Override by setting environment variable $TSAN_OPTIONS.
+extern "C" const char *__tsan_default_options() {
+ // Note that backend test should re-configure to halt_on_error=1
+ return "halt_on_error=0 history_size=7";
+}
+#endif
+
+// Default UBSAN_OPTIONS. Override by setting environment variable $UBSAN_OPTIONS.
+#if defined(UNDEFINED_SANITIZER)
+extern "C" const char *__ubsan_default_options() {
+ static const string default_options = Substitute(
+ "print_stacktrace=1 suppressions=$0/bin/ubsan-suppressions.txt",
+ getenv("IMPALA_HOME") == nullptr ? "." : getenv("IMPALA_HOME"));
+ return default_options.c_str();
+}
+#endif
http://git-wip-us.apache.org/repos/asf/impala/blob/2caec900/bin/run-backend-tests.sh
----------------------------------------------------------------------
diff --git a/bin/run-backend-tests.sh b/bin/run-backend-tests.sh
index d4d3142..3bc84c2 100755
--- a/bin/run-backend-tests.sh
+++ b/bin/run-backend-tests.sh
@@ -37,9 +37,7 @@ cd ${IMPALA_BE_DIR}
cd ..
export CTEST_OUTPUT_ON_FAILURE=1
-export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
-export UBSAN_OPTIONS="print_stacktrace=1"
-UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
+# Override default TSAN_OPTIONS so that halt_on_error is set.
export TSAN_OPTIONS="halt_on_error=1 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
"${MAKE_CMD:-make}" test ARGS="${BE_TEST_ARGS}"
http://git-wip-us.apache.org/repos/asf/impala/blob/2caec900/bin/start-catalogd.sh
----------------------------------------------------------------------
diff --git a/bin/start-catalogd.sh b/bin/start-catalogd.sh
index 4ec6846..a8b7e28 100755
--- a/bin/start-catalogd.sh
+++ b/bin/start-catalogd.sh
@@ -70,9 +70,5 @@ if ${CLUSTER_DIR}/admin is_kerberized; then
fi
. ${IMPALA_HOME}/bin/set-classpath.sh
-export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
-export UBSAN_OPTIONS="print_stacktrace=1"
-UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
-export TSAN_OPTIONS="halt_on_error=0 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/catalog/catalogd ${CATALOGD_ARGS}
http://git-wip-us.apache.org/repos/asf/impala/blob/2caec900/bin/start-impalad.sh
----------------------------------------------------------------------
diff --git a/bin/start-impalad.sh b/bin/start-impalad.sh
index 76a5f2c..f052fa2 100755
--- a/bin/start-impalad.sh
+++ b/bin/start-impalad.sh
@@ -98,9 +98,5 @@ if ${CLUSTER_DIR}/admin is_kerberized; then
fi
. ${IMPALA_HOME}/bin/set-classpath.sh
-export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
-export UBSAN_OPTIONS="print_stacktrace=1"
-UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
-export TSAN_OPTIONS="halt_on_error=0 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${TOOL_PREFIX} ${IMPALA_CMD} ${IMPALAD_ARGS}
http://git-wip-us.apache.org/repos/asf/impala/blob/2caec900/bin/start-statestored.sh
----------------------------------------------------------------------
diff --git a/bin/start-statestored.sh b/bin/start-statestored.sh
index 02cf09f..f023810 100755
--- a/bin/start-statestored.sh
+++ b/bin/start-statestored.sh
@@ -59,9 +59,5 @@ if ${CLUSTER_DIR}/admin is_kerberized; then
fi
fi
-export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
-export UBSAN_OPTIONS="print_stacktrace=1"
-UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
-export TSAN_OPTIONS="halt_on_error=0 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/statestore/statestored ${STATESTORED_ARGS}