You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by he...@apache.org on 2017/03/01 17:28:41 UTC
[2/6] incubator-impala git commit: Add a build flag for the undefined
behavior sanitizer, aka "ubsan".
Add a build flag for the undefined behavior sanitizer, aka "ubsan".
Ubsan checks for undefined behavior according to the C++
standard. Some of this behavior has been known to be exploited by
optimizing compilers to produce bizarre results, like taking both
branches of a conditional.
This patch only adds build options; fixing the errors ubsan finds, as
well as adding any tests that a build is free from ubsan errors, are
not covered in this patch.
Change-Id: I03044c657ac171daa0648f833bbbeed7bdde49cb
Reviewed-on: http://gerrit.cloudera.org:8080/6186
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/93eb8ccb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/93eb8ccb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/93eb8ccb
Branch: refs/heads/master
Commit: 93eb8ccbedda060af01dccbb452c28d8f2072115
Parents: 1417d76
Author: Jim Apple <jb...@apache.org>
Authored: Tue Feb 28 12:01:13 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 03:09:17 2017 +0000
----------------------------------------------------------------------
CMakeLists.txt | 3 ++-
be/CMakeLists.txt | 21 +++++++++++++++++++--
bin/make_impala.sh | 3 ++-
bin/run-backend-tests.sh | 3 +++
bin/start-impalad.sh | 2 ++
buildall.sh | 8 ++++++++
6 files changed, 36 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 80256cc..207fac4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -201,7 +201,8 @@ find_package(LlvmBinaries REQUIRED)
# Find LLVM libraries to link against.
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
- OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
# Use the LLVM libaries with assertions for debug builds.
set(LLVM_ROOT ${LLVM_DEBUG_ROOT})
endif()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 3697ceb..1e094a0 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -87,6 +87,19 @@ SET(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG -gdwarf-2")
SET(CXX_FLAGS_ADDRESS_SANITIZER
"${CXX_CLANG_FLAGS} -O1 -g -fsanitize=address -fno-omit-frame-pointer -DADDRESS_SANITIZER")
+# Set the flags to the undefined behavior sanitizer, also known as "ubsan"
+# Turn on sanitizer and debug symbols to get stack traces:
+SET(CXX_FLAGS_UBSAN "${CXX_CLANG_FLAGS} -ggdb3 -fno-omit-frame-pointer -fsanitize=undefined")
+# Add flags to enable symbol resolution in the stack traces:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -rtlib=compiler-rt -lgcc_s")
+# Ignore a number of noisy errors with too many false positives:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -fno-sanitize=alignment,function,vptr,float-divide-by-zero,float-cast-overflow")
+# Don't enforce wrapped signed integer arithmetic so that the sanitizer actually sees
+# undefined wrapping:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -fno-wrapv")
+# To ease debugging, turn off all optimizations:
+SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -O0")
+
SET(CXX_FLAGS_TIDY "${CXX_CLANG_FLAGS}")
# Catching unused variables requires an optimization level greater than 0
SET(CXX_FLAGS_TIDY "${CXX_FLAGS_TIDY} -O1")
@@ -111,6 +124,8 @@ elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_ADDRESS_SANITIZER}")
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_TIDY}")
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
+ SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_UBSAN}")
else()
message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
endif()
@@ -133,7 +148,8 @@ if (CCACHE AND NOT DEFINED ENV{DISABLE_CCACHE})
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
- OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
# Need to set CCACHE_CPP so that ccache calls clang with the original source file for
# both preprocessing and compilation. Otherwise, ccache will use clang to preprocess
# the file and then call clang with the preprocessed output if not cached. However,
@@ -247,7 +263,8 @@ add_definitions(-fPIC)
# set compile output directory
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
- "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")
+ "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" OR
+ "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/debug/")
else()
set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/release/")
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/make_impala.sh
----------------------------------------------------------------------
diff --git a/bin/make_impala.sh b/bin/make_impala.sh
index 70d088f..8d164c2 100755
--- a/bin/make_impala.sh
+++ b/bin/make_impala.sh
@@ -148,7 +148,8 @@ then
if [[ ! -z $IMPALA_TOOLCHAIN ]]; then
if [[ ("$TARGET_BUILD_TYPE" == "ADDRESS_SANITIZER") \
- || ("$TARGET_BUILD_TYPE" == "TIDY") ]]
+ || ("$TARGET_BUILD_TYPE" == "TIDY") \
+ || ("$TARGET_BUILD_TYPE" == "UBSAN") ]]
then
CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/clang_toolchain.cmake)
else
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/run-backend-tests.sh
----------------------------------------------------------------------
diff --git a/bin/run-backend-tests.sh b/bin/run-backend-tests.sh
index 758c1bc..f426542 100755
--- a/bin/run-backend-tests.sh
+++ b/bin/run-backend-tests.sh
@@ -34,6 +34,9 @@ fi
cd ${IMPALA_BE_DIR}
. ${IMPALA_HOME}/bin/set-classpath.sh
+cd ..
export CTEST_OUTPUT_ON_FAILURE=1
+export UBSAN_OPTIONS="print_stacktrace=1"
+export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
make test ARGS="${BE_TEST_ARGS}"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/bin/start-impalad.sh
----------------------------------------------------------------------
diff --git a/bin/start-impalad.sh b/bin/start-impalad.sh
index e7f2442..a731804 100755
--- a/bin/start-impalad.sh
+++ b/bin/start-impalad.sh
@@ -103,4 +103,6 @@ if ${CLUSTER_DIR}/admin is_kerberized; then
fi
. ${IMPALA_HOME}/bin/set-classpath.sh
+export UBSAN_OPTIONS='print_stacktrace=1'
+export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${TOOL_PREFIX} ${IMPALA_CMD} ${IMPALAD_ARGS}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/93eb8ccb/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index 9c4385c..bcc6ae3 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -59,6 +59,7 @@ CODE_COVERAGE=0
BUILD_ASAN=0
BUILD_FE_ONLY=0
BUILD_TIDY=0
+BUILD_UBSAN=0
# Export MAKE_CMD so it is visible in scripts that invoke make, e.g. copy-udfs-udas.sh
export MAKE_CMD=make
LZO_CMAKE_ARGS=
@@ -114,6 +115,9 @@ do
-tidy)
BUILD_TIDY=1
;;
+ -ubsan)
+ BUILD_UBSAN=1
+ ;;
-testpairwise)
EXPLORATION_STRATEGY=pairwise
;;
@@ -182,6 +186,7 @@ do
echo "[-codecoverage] : Build with code coverage [Default: False]"
echo "[-asan] : Address sanitizer build [Default: False]"
echo "[-tidy] : clang-tidy build [Default: False]"
+ echo "[-ubsan] : Undefined behavior build [Default: False]"
echo "[-skiptests] : Skips execution of all tests"
echo "[-notests] : Skips building and execution of all tests"
echo "[-start_minicluster] : Start test cluster including Impala and all"\
@@ -260,6 +265,9 @@ fi
if [[ ${BUILD_TIDY} -eq 1 ]]; then
CMAKE_BUILD_TYPE=TIDY
fi
+if [[ ${BUILD_UBSAN} -eq 1 ]]; then
+ CMAKE_BUILD_TYPE=UBSAN
+fi
MAKE_IMPALA_ARGS+=" -build_type=${CMAKE_BUILD_TYPE}"