You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/09/29 17:50:19 UTC
[4/7] kudu git commit: thirdparty: split into dependency groups
thirdparty: split into dependency groups
The monolithic thirdparty build is now quite a bit larger than it used to be
on account of the extra LLVM build. Let's see if we can't speed it up. The
idea is simple: carve it up into disjoint sections so that individual
sections can be rebuilt as needed.
This patch separates the various portions of the thirdparty build into
"dependency groups". Conceptually, a dependency group is a set of
dependencies built a certain way, but the implementation is really just a
set of non-overlapping code fragments in build-thirdparty.sh.
The initial set of groups are:
- common: dependencies that are never instrumented.
- uninstrumented: dependencies that may be instrumented, but aren't in this
build.
- tsan: dependencies that may be instrumented, and are indeed in this build
(with -fsanitize=thread).
These three generally map to the existing "common", "uninstrumented", and
"tsan" thirdparty subdirectories. There's an obvious pattern here for future
sanitizer builds (e.g. MSAN would provide an "msan" dependency group).
The new build-if-necessary.sh can accept an argument that maps to a set of
dependency groups representing a particular build. Every dependency group
has its own hash/stamp file so that it is only rebuilt when needed.
This also fixes a bug in the stamp file approach that prevented it from
actually rebuilding anything.
Change-Id: I549262858f98b5ce6c78e786e8c8d8134ba2ed36
Reviewed-on: http://gerrit.cloudera.org:8080/4513
Reviewed-by: Dan Burkert <da...@cloudera.com>
Tested-by: Dan Burkert <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/b208e630
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/b208e630
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/b208e630
Branch: refs/heads/master
Commit: b208e6303d6a3edc08d73cbb075d55c4c321cc3f
Parents: 3625a0c
Author: Adar Dembo <ad...@cloudera.com>
Authored: Tue Sep 20 15:11:33 2016 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Thu Sep 29 17:47:29 2016 +0000
----------------------------------------------------------------------
CMakeLists.txt | 5 +-
build-support/jenkins/build-and-test.sh | 6 +-
thirdparty/.gitignore | 2 +-
thirdparty/build-if-necessary.sh | 102 ++++++---
thirdparty/build-thirdparty.sh | 324 ++++++++++++++-------------
5 files changed, 254 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/b208e630/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3c01ef7..754aa63 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -102,8 +102,11 @@ set(CMAKE_ENABLE_EXPORTS true)
# Make sure thirdparty stuff is up-to-date.
if ("$ENV{NO_REBUILD_THIRDPARTY}" STREQUAL "")
+ if (${KUDU_USE_TSAN})
+ set(TP_ARGS "tsan")
+ endif()
execute_process(
- COMMAND ${THIRDPARTY_DIR}/build-if-necessary.sh
+ COMMAND ${THIRDPARTY_DIR}/build-if-necessary.sh ${TP_ARGS}
RESULT_VARIABLE THIRDPARTY_SCRIPT_RESULT)
if (NOT (${THIRDPARTY_SCRIPT_RESULT} EQUAL 0))
message(FATAL_ERROR "Thirdparty was built unsuccessfully, terminating.")
http://git-wip-us.apache.org/repos/asf/kudu/blob/b208e630/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index 6725858..79bea7d 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -145,7 +145,11 @@ if [ -d "$TOOLCHAIN_DIR" ]; then
PATH=$TOOLCHAIN_DIR/apache-maven-3.0/bin:$PATH
fi
-$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh
+THIRDPARTY_TYPE=
+if [ "$BUILD_TYPE" = "TSAN" ]; then
+ THIRDPARTY_TYPE=tsan
+fi
+$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh $THIRDPARTY_TYPE
THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof
http://git-wip-us.apache.org/repos/asf/kudu/blob/b208e630/thirdparty/.gitignore
----------------------------------------------------------------------
diff --git a/thirdparty/.gitignore b/thirdparty/.gitignore
index ea9ed85..afd3df7 100644
--- a/thirdparty/.gitignore
+++ b/thirdparty/.gitignore
@@ -18,5 +18,5 @@
installed/
src/
build/
-.build-hash
+.build-hash*
clang-toolchain
http://git-wip-us.apache.org/repos/asf/kudu/blob/b208e630/thirdparty/build-if-necessary.sh
----------------------------------------------------------------------
diff --git a/thirdparty/build-if-necessary.sh b/thirdparty/build-if-necessary.sh
index 8239b85..03f06cd 100755
--- a/thirdparty/build-if-necessary.sh
+++ b/thirdparty/build-if-necessary.sh
@@ -26,61 +26,103 @@
set -e
set -o pipefail
+DEPENDENCY_GROUPS=
+case $1 in
+ "")
+ DEPENDENCY_GROUPS="common uninstrumented"
+ ;;
+ "tsan")
+ DEPENDENCY_GROUPS="common tsan"
+ ;;
+ "all")
+ DEPENDENCY_GROUPS="common uninstrumented tsan"
+ ;;
+ *)
+ echo "Unknown build configuration: $1"
+ exit 1
+ ;;
+esac
+
+
TP_DIR=$(dirname $BASH_SOURCE)
cd $TP_DIR
NEEDS_BUILD=
+NEEDS_REHASH=
IS_IN_GIT=$(test -d ../.git && echo true || :)
if [ -n "$IS_IN_GIT" ]; then
# Determine whether this subtree in the git repo has changed since thirdparty
- # was last built
-
+ # was last built.
CUR_THIRDPARTY_HASH=$(cd .. && git ls-tree -d HEAD thirdparty | awk '{print $3}')
- LAST_BUILD_HASH=$(cat .build-hash || :)
- if [ "$CUR_THIRDPARTY_HASH" != "$LAST_BUILD_HASH" ]; then
- echo "Rebuilding thirdparty: the repository has changed since thirdparty was last built."
- echo "Old git hash: $LAST_BUILD_HASH"
- echo "New build hash: $CUR_THIRDPARTY_HASH"
- NEEDS_BUILD=1
- else
- # Determine whether the developer has any local changes
+
+ for GROUP in $DEPENDENCY_GROUPS; do
+ LAST_BUILD_HASH=$(cat .build-hash.$GROUP || :)
+ if [ "$CUR_THIRDPARTY_HASH" != "$LAST_BUILD_HASH" ]; then
+ echo "Rebuilding thirdparty dependency group '$GROUP': the repository has changed since it was last built."
+ echo "Old git hash: $LAST_BUILD_HASH"
+ echo "New build hash: $CUR_THIRDPARTY_HASH"
+ NEEDS_BUILD="$NEEDS_BUILD $GROUP"
+ NEEDS_REHASH="$NEEDS_REHASH $GROUP"
+ fi
+ done
+
+ if [ -z "$NEEDS_BUILD" ]; then
+ # All the hashes matched. Determine whether the developer has any local changes.
if ! ( git diff --quiet . && git diff --cached --quiet . ) ; then
- echo "Rebuilding thirdparty: There are local changes in the repository."
- NEEDS_BUILD=1
- else
- echo Not rebuilding thirdparty. No changes since last build.
+ echo "Rebuilding thirdparty dependency groups '$DEPENDENCY_GROUPS': there are local changes in the repository."
+ NEEDS_BUILD="$DEPENDENCY_GROUPS"
fi
fi
+
+ if [ -z "$NEEDS_BUILD" ]; then
+ echo "Not rebuilding thirdparty. No changes since last build."
+ fi
else
# If we aren't inside running inside a git repository (e.g. we are
# part of a source distribution tarball) then we can't use git to find
- # out whether the build is clean. Instead, use a .build-stamp file, and
- # see if any files inside this directory have been modified since then.
- if [ -f .build-stamp ]; then
- CHANGED_FILE_COUNT=$(find . -cnewer .build-stamp | wc -l)
- echo "$CHANGED_FILE_COUNT file(s) been modified since thirdparty was last built."
- if [ $CHANGED_FILE_COUNT -gt 0 ]; then
- echo "Rebuilding."
- echo NEEDS_BUILD=1
+ # out whether the build is clean. Instead, look at the ctimes of special
+ # stamp files, and see if any files inside this directory have been
+ # modified since then.
+ for GROUP in $DEPENDENCY_GROUPS; do
+ STAMP_FILE=.build-stamp.$GROUP
+ if [ -f $STAMP_FILE ]; then
+ CHANGED_FILE_COUNT=$(find . -cnewer $STAMP_FILE | wc -l)
+ echo "$CHANGED_FILE_COUNT file(s) been modified since thirdparty dependency group '$GROUP' was last built."
+ if [ $CHANGED_FILE_COUNT -gt 0 ]; then
+ echo "Rebuilding."
+ NEEDS_BUILD="$NEEDS_BUILD $GROUP"
+ fi
+ else
+ echo "It appears that thirdparty dependency group '$GROUP' was never built. Building."
+ NEEDS_BUILD="$NEEDS_BUILD $GROUP"
fi
- else
- echo "It appears that thirdparty was never built. Building."
- NEEDS_BUILD=1
- fi
+ done
fi
if [ -z "$NEEDS_BUILD" ]; then
exit 0
fi
-rm -f .build-hash .build-stamp
+# Remove the old hashes/stamps before building so that if the build is aborted
+# and the repository is taken backwards in time to the point where the old
+# hashes/stamps matched, the repository would still be rebuilt.
+for GROUP in $NEEDS_BUILD; do
+ rm -f .build-hash.$GROUP .build-stamp.$GROUP
+done
+
+# Download and build the necessary dependency groups.
./download-thirdparty.sh
-./build-thirdparty.sh
+./build-thirdparty.sh $NEEDS_BUILD
+# The build succeeded. Update the appropriate hashes/stamps.
if [ -n "$IS_IN_GIT" ]; then
- echo $CUR_THIRDPARTY_HASH > .build-hash
+ for GROUP in $NEEDS_REHASH; do
+ echo $CUR_THIRDPARTY_HASH > .build-hash.$GROUP
+ done
else
- touch .build-stamp
+ for GROUP in $NEEDS_BUILD; do
+ touch .build-stamp.$GROUP
+ done
fi
http://git-wip-us.apache.org/repos/asf/kudu/blob/b208e630/thirdparty/build-thirdparty.sh
----------------------------------------------------------------------
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 6271b3b..bfbfbf1 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -46,6 +46,56 @@ source $TP_DIR/build-definitions.sh
# read the docs)
$TP_DIR/preflight.py
+################################################################################
+
+if [ "$#" = "0" ]; then
+ ARGS_TO_PRINT="common uninstrumented tsan"
+
+ F_COMMON=1
+ F_UNINSTRUMENTED=1
+ F_TSAN=1
+else
+ ARGS_TO_PRINT="$*"
+ REQUESTED_EXPLICIT_DEPENDENCIES=1
+
+ # Parse the command line for specific dependencies or dependency groups.
+ for arg in $*; do
+ case $arg in
+ # Dependency groups.
+ "common") F_COMMON=1 ;;
+ "uninstrumented") F_UNINSTRUMENTED=1 ;;
+ "tsan") F_TSAN=1 ;;
+
+ # Dependencies.
+ "cmake") F_CMAKE=1 ;;
+ "gflags") F_GFLAGS=1 ;;
+ "glog") F_GLOG=1 ;;
+ "gmock") F_GMOCK=1 ;;
+ "gperftools") F_GPERFTOOLS=1 ;;
+ "libev") F_LIBEV=1 ;;
+ "lz4") F_LZ4=1 ;;
+ "bitshuffle") F_BITSHUFFLE=1 ;;
+ "protobuf") F_PROTOBUF=1 ;;
+ "rapidjson") F_RAPIDJSON=1 ;;
+ "snappy") F_SNAPPY=1 ;;
+ "zlib") F_ZLIB=1 ;;
+ "squeasel") F_SQUEASEL=1 ;;
+ "gsg") F_GSG=1 ;;
+ "gcovr") F_GCOVR=1 ;;
+ "curl") F_CURL=1 ;;
+ "crcutil") F_CRCUTIL=1 ;;
+ "libunwind") F_LIBUNWIND=1 ;;
+ "llvm") F_LLVM=1 ;;
+ "trace-viewer") F_TRACE_VIEWER=1 ;;
+ "nvml") F_NVML=1 ;;
+ "boost") F_BOOST=1 ;;
+ *) echo "Unknown module: $arg"; exit 1 ;;
+ esac
+ done
+fi
+
+################################################################################
+
for PREFIX_DIR in $PREFIX_COMMON $PREFIX_DEPS $PREFIX_DEPS_TSAN; do
mkdir -p $PREFIX_DIR/lib
mkdir -p $PREFIX_DIR/include
@@ -80,9 +130,6 @@ if [[ "$OSTYPE" =~ ^linux ]]; then
# 2. https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html
EXTRA_CXXFLAGS="$EXTRA_CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
DYLIB_SUFFIX="so"
-
- # Enable TSAN builds on Linux.
- F_TSAN=1
elif [[ "$OSTYPE" == "darwin"* ]]; then
OS_OSX=1
DYLIB_SUFFIX="dylib"
@@ -94,48 +141,22 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
EXTRA_CXXFLAGS="$EXTRA_CXXFLAGS -stdlib=libc++"
EXTRA_LDFLAGS="$EXTRA_LDFLAGS -stdlib=libc++"
EXTRA_LIBS="$EXTRA_LIBS -lc++ -lc++abi"
+
+ # TSAN doesn't work on macOS. If it was explicitly asked for, respond with an
+ # error. Otherwise, just disable it silently.
+ if [ -n "$F_TSAN" ]; then
+ if [ -n "$REQUESTED_EXPLICIT_DEPENDENCIES" ]; then
+ echo TSAN does not work on macOS
+ exit 1
+ else
+ unset F_TSAN
+ fi
+ fi
else
echo Unsupported platform $OSTYPE
exit 1
fi
-################################################################################
-
-if [ "$#" = "0" ]; then
- F_ALL=1
-else
- # Allow passing specific libs to build on the command line
- for arg in $*; do
- case $arg in
- "cmake") F_CMAKE=1 ;;
- "gflags") F_GFLAGS=1 ;;
- "glog") F_GLOG=1 ;;
- "gmock") F_GMOCK=1 ;;
- "gperftools") F_GPERFTOOLS=1 ;;
- "libev") F_LIBEV=1 ;;
- "lz4") F_LZ4=1 ;;
- "bitshuffle") F_BITSHUFFLE=1;;
- "protobuf") F_PROTOBUF=1 ;;
- "rapidjson") F_RAPIDJSON=1 ;;
- "snappy") F_SNAPPY=1 ;;
- "zlib") F_ZLIB=1 ;;
- "squeasel") F_SQUEASEL=1 ;;
- "gsg") F_GSG=1 ;;
- "gcovr") F_GCOVR=1 ;;
- "curl") F_CURL=1 ;;
- "crcutil") F_CRCUTIL=1 ;;
- "libunwind") F_LIBUNWIND=1 ;;
- "llvm") F_LLVM=1 ;;
- "trace-viewer") F_TRACE_VIEWER=1 ;;
- "nvml") F_NVML=1 ;;
- "boost") F_BOOST=1 ;;
- *) echo "Unknown module: $arg"; exit 1 ;;
- esac
- done
-fi
-
-################################################################################
-
### Build common tools and libraries
PREFIX=$PREFIX_COMMON
@@ -144,7 +165,7 @@ MODE_SUFFIX=""
# Add tools to path
export PATH=$PREFIX/bin:$PATH
-if [ -n "$F_ALL" -o -n "$F_CMAKE" ]; then
+if [ -n "$F_COMMON" -o -n "$F_CMAKE" ]; then
build_cmake
fi
@@ -167,55 +188,55 @@ EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"
# add $PREFIX/lib to -Wl,-rpath.
EXTRA_LDFLAGS="-Wl,-rpath,$PREFIX/lib $EXTRA_LDFLAGS"
-if [ -n "$OS_LINUX" ] && [ -n "$F_ALL" -o -n "$F_LIBUNWIND" ]; then
+if [ -n "$OS_LINUX" ] && [ -n "$F_COMMON" -o -n "$F_LIBUNWIND" ]; then
build_libunwind
fi
-if [ -n "$F_ALL" -o -n "$F_ZLIB" ]; then
+if [ -n "$F_COMMON" -o -n "$F_ZLIB" ]; then
build_zlib
fi
-if [ -n "$F_ALL" -o -n "$F_LZ4" ]; then
+if [ -n "$F_COMMON" -o -n "$F_LZ4" ]; then
build_lz4
fi
-if [ -n "$F_ALL" -o -n "$F_BITSHUFFLE" ]; then
+if [ -n "$F_COMMON" -o -n "$F_BITSHUFFLE" ]; then
build_bitshuffle
fi
-if [ -n "$F_ALL" -o -n "$F_LIBEV" ]; then
+if [ -n "$F_COMMON" -o -n "$F_LIBEV" ]; then
build_libev
fi
-if [ -n "$F_ALL" -o -n "$F_RAPIDJSON" ]; then
+if [ -n "$F_COMMON" -o -n "$F_RAPIDJSON" ]; then
build_rapidjson
fi
-if [ -n "$F_ALL" -o -n "$F_SQUEASEL" ]; then
+if [ -n "$F_COMMON" -o -n "$F_SQUEASEL" ]; then
build_squeasel
fi
-if [ -n "$F_ALL" -o -n "$F_CURL" ]; then
+if [ -n "$F_COMMON" -o -n "$F_CURL" ]; then
build_curl
fi
-if [ -n "$F_ALL" -o -n "$F_GSG" ]; then
+if [ -n "$F_COMMON" -o -n "$F_GSG" ]; then
build_cpplint
fi
-if [ -n "$F_ALL" -o -n "$F_GCOVR" ]; then
+if [ -n "$F_COMMON" -o -n "$F_GCOVR" ]; then
build_gcovr
fi
-if [ -n "$F_ALL" -o -n "$F_TRACE_VIEWER" ]; then
+if [ -n "$F_COMMON" -o -n "$F_TRACE_VIEWER" ]; then
build_trace_viewer
fi
-if [ -n "$OS_LINUX" ] && [ -n "$F_ALL" -o -n "$F_NVML" ]; then
+if [ -n "$OS_LINUX" ] && [ -n "$F_COMMON" -o -n "$F_NVML" ]; then
build_nvml
fi
-if [ -n "$F_ALL" -o -n "$F_BOOST" ]; then
+if [ -n "$F_COMMON" -o -n "$F_BOOST" ]; then
build_boost
fi
@@ -230,7 +251,9 @@ save_env
EXTRA_LDFLAGS="-Wl,-rpath,$PREFIX/lib $EXTRA_LDFLAGS"
-if [ -n "$F_ALL" -o -n "$F_LLVM" ]; then
+# Clang is used by all builds so it is part of the 'common' library group even
+# though its LLVM libraries are installed to $PREFIX_DEPS.
+if [ -n "$F_COMMON" -o -n "$F_LLVM" ]; then
build_llvm normal
fi
@@ -240,31 +263,31 @@ fi
EXTRA_CFLAGS="-g $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"
-if [ -n "$F_ALL" -o -n "$F_GFLAGS" ]; then
+if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GFLAGS" ]; then
build_gflags
fi
-if [ -n "$F_ALL" -o -n "$F_GLOG" ]; then
+if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GLOG" ]; then
build_glog
fi
-if [ -n "$F_ALL" -o -n "$F_GPERFTOOLS" ]; then
+if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GPERFTOOLS" ]; then
build_gperftools
fi
-if [ -n "$F_ALL" -o -n "$F_GMOCK" ]; then
+if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GMOCK" ]; then
build_gmock
fi
-if [ -n "$F_ALL" -o -n "$F_PROTOBUF" ]; then
+if [ -n "$F_UNINSTRUMENTED" -o -n "$F_PROTOBUF" ]; then
build_protobuf
fi
-if [ -n "$F_ALL" -o -n "$F_SNAPPY" ]; then
+if [ -n "$F_UNINSTRUMENTED" -o -n "$F_SNAPPY" ]; then
build_snappy
fi
-if [ -n "$F_ALL" -o -n "$F_CRCUTIL" ]; then
+if [ -n "$F_UNINSTRUMENTED" -o -n "$F_CRCUTIL" ]; then
build_crcutil
fi
@@ -272,115 +295,112 @@ restore_env
## Build C++ dependencies with TSAN instrumentation
-if [ -n "$F_TSAN" ]; then
-
- # Achieving good results with TSAN requires that:
- # 1. The C++ standard library should be instrumented with TSAN.
- # 2. Dependencies which internally use threads or synchronization be
- # instrumented with TSAN.
- # 3. As a corollary to 1, the C++ standard library requires that all shared
- # objects linked into an executable be built against the same version of
- # the C++ standard library version.
- #
- # At the very least, we must build our own C++ standard library. We use libc++
- # because it's easy to build with clang, which has better TSAN support than gcc.
- #
- # To satisfy all of the above requirements, we first build libc++ instrumented
- # with TSAN, then build a second copy of every C++ dependency against that
- # libc++. Later on in the build process, Kudu is also built against libc++.
- #
- # Special flags for TSAN builds:
- # * -fsanitize=thread - enable the thread sanitizer during compilation.
- # * -L ... - add the instrumented libc++ to the library search paths.
- # * -isystem ... - Add libc++ headers to the system header search paths.
- # * -nostdinc++ - Do not automatically link the system C++ standard library.
- # * -Wl,-rpath,... - Add instrumented libc++ location to the rpath so that
- # it can be found at runtime.
-
- if which ccache >/dev/null ; then
- CLANG="$TP_DIR/../build-support/ccache-clang/clang"
- CLANGXX="$TP_DIR/../build-support/ccache-clang/clang++"
- else
- CLANG="$TP_DIR/clang-toolchain/bin/clang"
- CLANGXX="$TP_DIR/clang-toolchain/bin/clang++"
- fi
- export CC=$CLANG
- export CXX=$CLANGXX
-
- PREFIX=$PREFIX_DEPS_TSAN
- MODE_SUFFIX=".tsan"
+# Achieving good results with TSAN requires that:
+# 1. The C++ standard library should be instrumented with TSAN.
+# 2. Dependencies which internally use threads or synchronization be
+# instrumented with TSAN.
+# 3. As a corollary to 1, the C++ standard library requires that all shared
+# objects linked into an executable be built against the same version of the
+# C++ standard library version.
+#
+# At the very least, we must build our own C++ standard library. We use libc++
+# because it's easy to build with clang, which has better TSAN support than gcc.
+#
+# To satisfy all of the above requirements, we first build libc++ instrumented
+# with TSAN, then build a second copy of every C++ dependency against that
+# libc++. Later on in the build process, Kudu is also built against libc++.
+#
+# Special flags for TSAN builds:
+# * -fsanitize=thread - enable the thread sanitizer during compilation.
+# * -L ... - add the instrumented libc++ to the library search paths.
+# * -isystem ... - Add libc++ headers to the system header search paths.
+# * -nostdinc++ - Do not automatically link the system C++ standard library.
+# * -Wl,-rpath,... - Add instrumented libc++ location to the rpath so that it
+# can be found at runtime.
+
+if which ccache >/dev/null ; then
+ CLANG="$TP_DIR/../build-support/ccache-clang/clang"
+ CLANGXX="$TP_DIR/../build-support/ccache-clang/clang++"
+else
+ CLANG="$TP_DIR/clang-toolchain/bin/clang"
+ CLANGXX="$TP_DIR/clang-toolchain/bin/clang++"
+fi
+export CC=$CLANG
+export CXX=$CLANGXX
- save_env
+PREFIX=$PREFIX_DEPS_TSAN
+MODE_SUFFIX=".tsan"
- # Build libc++abi first as it is a dependency for libc++. Its build has no
- # built-in support for sanitizers, so we build it regularly.
- if [ -n "$F_ALL" -o -n "$F_LLVM" ]; then
- build_libcxxabi
- fi
+save_env
- # The libc++ build needs to be able to find libc++abi.
- EXTRA_CXXFLAGS="-L$PREFIX/lib $EXTRA_CXXFLAGS"
- EXTRA_LDFLAGS="-Wl,-rpath,$PREFIX/lib $EXTRA_LDFLAGS"
+# Build libc++abi first as it is a dependency for libc++. Its build has no
+# built-in support for sanitizers, so we build it regularly.
+if [ -n "$F_TSAN" -o -n "$F_LLVM" ]; then
+ build_libcxxabi
+fi
- # Build libc++ with TSAN enabled.
- if [ -n "$F_ALL" -o -n "$F_LLVM" ]; then
- build_libcxx tsan
- fi
+# The libc++ build needs to be able to find libc++abi.
+EXTRA_CXXFLAGS="-L$PREFIX/lib $EXTRA_CXXFLAGS"
+EXTRA_LDFLAGS="-Wl,-rpath,$PREFIX/lib $EXTRA_LDFLAGS"
- # Build the rest of the dependencies against the TSAN-instrumented libc++
- # instead of the system's C++ standard library.
- EXTRA_CXXFLAGS="-nostdinc++ $EXTRA_CXXFLAGS"
- EXTRA_CXXFLAGS="-stdlib=libc++ $EXTRA_CXXFLAGS"
- EXTRA_CXXFLAGS="-isystem $PREFIX/include/c++/v1 $EXTRA_CXXFLAGS"
+# Build libc++ with TSAN enabled.
+if [ -n "$F_TSAN" -o -n "$F_LLVM" ]; then
+ build_libcxx tsan
+fi
- # Build the rest of the dependencies with TSAN instrumentation.
- EXTRA_CFLAGS="-fsanitize=thread $EXTRA_CFLAGS"
- EXTRA_CXXFLAGS="-fsanitize=thread $EXTRA_CXXFLAGS"
- EXTRA_CXXFLAGS="-DTHREAD_SANITIZER $EXTRA_CXXFLAGS"
+# Build the rest of the dependencies against the TSAN-instrumented libc++
+# instead of the system's C++ standard library.
+EXTRA_CXXFLAGS="-nostdinc++ $EXTRA_CXXFLAGS"
+EXTRA_CXXFLAGS="-stdlib=libc++ $EXTRA_CXXFLAGS"
+EXTRA_CXXFLAGS="-isystem $PREFIX/include/c++/v1 $EXTRA_CXXFLAGS"
- if [ -n "$F_ALL" -o -n "$F_LLVM" ]; then
- build_llvm tsan
- fi
+# Build the rest of the dependencies with TSAN instrumentation.
+EXTRA_CFLAGS="-fsanitize=thread $EXTRA_CFLAGS"
+EXTRA_CXXFLAGS="-fsanitize=thread $EXTRA_CXXFLAGS"
+EXTRA_CXXFLAGS="-DTHREAD_SANITIZER $EXTRA_CXXFLAGS"
- # Enable debug symbols so that stacktraces and linenumbers are available at
- # runtime. LLVM is compiled without debug symbols because the LLVM debug symbols
- # take up more than 20GiB of disk space.
- EXTRA_CFLAGS="-g $EXTRA_CFLAGS"
- EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"
+if [ -n "$F_TSAN" -o -n "$F_LLVM" ]; then
+ build_llvm tsan
+fi
- if [ -n "$F_ALL" -o -n "$F_PROTOBUF" ]; then
- build_protobuf
- fi
+# Enable debug symbols so that stacktraces and linenumbers are available at
+# runtime. LLVM is compiled without debug symbols because the LLVM debug symbols
+# take up more than 20GiB of disk space.
+EXTRA_CFLAGS="-g $EXTRA_CFLAGS"
+EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"
- if [ -n "$F_ALL" -o -n "$F_GFLAGS" ]; then
- build_gflags
- fi
+if [ -n "$F_TSAN" -o -n "$F_PROTOBUF" ]; then
+ build_protobuf
+fi
- if [ -n "$F_ALL" -o -n "$F_GLOG" ]; then
- build_glog
- fi
+if [ -n "$F_TSAN" -o -n "$F_GFLAGS" ]; then
+ build_gflags
+fi
- if [ -n "$F_ALL" -o -n "$F_GPERFTOOLS" ]; then
- build_gperftools
- fi
+if [ -n "$F_TSAN" -o -n "$F_GLOG" ]; then
+ build_glog
+fi
- if [ -n "$F_ALL" -o -n "$F_GMOCK" ]; then
- build_gmock
- fi
+if [ -n "$F_TSAN" -o -n "$F_GPERFTOOLS" ]; then
+ build_gperftools
+fi
- if [ -n "$F_ALL" -o -n "$F_SNAPPY" ]; then
- build_snappy
- fi
+if [ -n "$F_TSAN" -o -n "$F_GMOCK" ]; then
+ build_gmock
+fi
- if [ -n "$F_ALL" -o -n "$F_CRCUTIL" ]; then
- build_crcutil
- fi
+if [ -n "$F_TSAN" -o -n "$F_SNAPPY" ]; then
+ build_snappy
+fi
- restore_env
+if [ -n "$F_TSAN" -o -n "$F_CRCUTIL" ]; then
+ build_crcutil
fi
+restore_env
+
# Now run the post-flight checks.
$TP_DIR/postflight.py
echo "---------------------"
-echo "Thirdparty dependencies built and installed into $PREFIX successfully"
+echo "Thirdparty dependencies '$ARGS_TO_PRINT' built and installed successfully"