You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by ad...@apache.org on 2016/09/28 02:22:49 UTC
kudu git commit: thirdparty: upgrade LLVM to 3.9.0
Repository: kudu
Updated Branches:
refs/heads/master 591e6ddfd -> dfe96efda
thirdparty: upgrade LLVM to 3.9.0
In this release of LLVM we're also including the libc++ and libc++abi
projects in the tarball for the first time. They will be used in upcoming
patches for building TSAN.
Of note, this release includes native devtoolset support. See clang commit
2fcb443.
I have no idea what I'm doing in the codegen module. But unit tests pass.
I also included a nasty hack in the TSAN-enabled libstdcxx build to ensure
that the final product has std::__once and std::__once_callable. AFAICT,
non-devtoolset distros have never produced a libstdcxx that defines those
symbols, but that hasn't been a problem until this LLVM upgrade, as the new
LLVM depends on them.
Change-Id: I7b4a19c9acb0ee76e3a27186fabb30288a28be2c
Reviewed-on: http://gerrit.cloudera.org:8080/4507
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <da...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/dfe96efd
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/dfe96efd
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/dfe96efd
Branch: refs/heads/master
Commit: dfe96efda39dfa13a68c7465a4cd963275559baa
Parents: 591e6dd
Author: Adar Dembo <ad...@cloudera.com>
Authored: Fri Sep 2 17:33:16 2016 -0700
Committer: Adar Dembo <ad...@cloudera.com>
Committed: Wed Sep 28 02:17:26 2016 +0000
----------------------------------------------------------------------
src/kudu/codegen/code_generator.cc | 2 +-
src/kudu/codegen/module_builder.cc | 18 ++++++++++++------
src/kudu/codegen/module_builder.h | 8 ++++----
thirdparty/build-definitions.sh | 13 +++++++++++++
thirdparty/download-thirdparty.sh | 3 +--
.../patches/llvm-devtoolset-toolchain.patch | 16 ----------------
thirdparty/vars.sh | 12 +++++++-----
7 files changed, 38 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kudu/blob/dfe96efd/src/kudu/codegen/code_generator.cc
----------------------------------------------------------------------
diff --git a/src/kudu/codegen/code_generator.cc b/src/kudu/codegen/code_generator.cc
index 6d3eba1..337321b 100644
--- a/src/kudu/codegen/code_generator.cc
+++ b/src/kudu/codegen/code_generator.cc
@@ -28,7 +28,7 @@
#include <llvm/IR/Instructions.h>
#include <llvm/IR/Module.h>
#include <llvm/MC/MCContext.h>
-#include <llvm/MC/MCDisassembler.h>
+#include <llvm/MC/MCDisassembler/MCDisassembler.h>
#include <llvm/MC/MCInst.h>
#include <llvm/MC/MCInstPrinter.h>
#include <llvm/Support/raw_os_ostream.h>
http://git-wip-us.apache.org/repos/asf/kudu/blob/dfe96efd/src/kudu/codegen/module_builder.cc
----------------------------------------------------------------------
diff --git a/src/kudu/codegen/module_builder.cc b/src/kudu/codegen/module_builder.cc
index 7d446c3..9f825e6 100644
--- a/src/kudu/codegen/module_builder.cc
+++ b/src/kudu/codegen/module_builder.cc
@@ -20,6 +20,7 @@
#include <cstdlib>
#include <sstream>
#include <string>
+#include <unordered_set>
#include <vector>
#include <glog/logging.h>
@@ -29,6 +30,7 @@
#include <llvm/ExecutionEngine/MCJIT.h>
#include <llvm/IR/DerivedTypes.h>
#include <llvm/IR/Function.h>
+#include <llvm/IR/GlobalValue.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/LegacyPassManager.h>
#include <llvm/IR/LLVMContext.h>
@@ -43,6 +45,7 @@
#include "kudu/codegen/precompiled.ll.h"
#include "kudu/gutil/macros.h"
+#include "kudu/gutil/map-util.h"
#include "kudu/gutil/strings/substitute.h"
#include "kudu/util/status.h"
@@ -61,6 +64,7 @@ using llvm::EngineBuilder;
using llvm::ExecutionEngine;
using llvm::Function;
using llvm::FunctionType;
+using llvm::GlobalValue;
using llvm::IntegerType;
using llvm::legacy::FunctionPassManager;
using llvm::legacy::PassManager;
@@ -78,6 +82,7 @@ using std::ostream;
using std::ostringstream;
using std::string;
using std::unique_ptr;
+using std::unordered_set;
using std::vector;
using strings::Substitute;
@@ -209,7 +214,7 @@ namespace {
void DoOptimizations(ExecutionEngine* engine,
Module* module,
- const vector<const char*>& external_functions) {
+ const unordered_set<string>& external_functions) {
PassManagerBuilder pass_builder;
pass_builder.OptLevel = 2;
// Don't optimize for code size (this corresponds to -O2/-O3)
@@ -231,7 +236,9 @@ void DoOptimizations(ExecutionEngine* engine,
PassManager module_passes;
// Internalize all functions that aren't explicitly specified with external linkage.
- module_passes.add(llvm::createInternalizePass(external_functions));
+ module_passes.add(llvm::createInternalizePass([&](const GlobalValue& v) {
+ return ContainsKey(external_functions, v.getGlobalIdentifier());
+ }));
pass_builder.populateModulePassManager(module_passes);
// Same as above, the result here just indicates whether optimization made any changes.
@@ -316,11 +323,10 @@ TargetMachine* ModuleBuilder::GetTargetMachine() const {
return CHECK_NOTNULL(target_);
}
-vector<const char*> ModuleBuilder::GetFunctionNames() const {
- vector<const char*> ret;
+unordered_set<string> ModuleBuilder::GetFunctionNames() const {
+ unordered_set<string> ret;
for (const JITFuture& fut : futures_) {
- const char* name = CHECK_NOTNULL(fut.llvm_f_)->getName().data();
- ret.push_back(name);
+ ret.insert(CHECK_NOTNULL(fut.llvm_f_)->getName());
}
return ret;
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/dfe96efd/src/kudu/codegen/module_builder.h
----------------------------------------------------------------------
diff --git a/src/kudu/codegen/module_builder.h b/src/kudu/codegen/module_builder.h
index a5661b0..4b500dd 100644
--- a/src/kudu/codegen/module_builder.h
+++ b/src/kudu/codegen/module_builder.h
@@ -20,6 +20,7 @@
#include <memory>
#include <string>
+#include <unordered_set>
#include <vector>
#include <llvm/IR/IRBuilder.h>
@@ -131,10 +132,9 @@ class ModuleBuilder {
};
void AddJITPromise(llvm::Function* llvm_f, FunctionAddress* actual_f);
- // Returns a vector of the function names for the functions stored in the
- // JITFutures. The pointers are valid so long as the futures_ vector's
- // elements have valid llvm::Function* values.
- std::vector<const char*> GetFunctionNames() const;
+
+ // Returns the function names for the functions stored in the JITFutures.
+ std::unordered_set<std::string> GetFunctionNames() const;
MBState state_;
std::vector<JITFuture> futures_;
http://git-wip-us.apache.org/repos/asf/kudu/blob/dfe96efd/thirdparty/build-definitions.sh
----------------------------------------------------------------------
diff --git a/thirdparty/build-definitions.sh b/thirdparty/build-definitions.sh
index 9e6e970..222885c 100644
--- a/thirdparty/build-definitions.sh
+++ b/thirdparty/build-definitions.sh
@@ -87,6 +87,8 @@ build_llvm() {
-DCMAKE_INSTALL_PREFIX=$PREFIX \
-DLLVM_TARGETS_TO_BUILD=X86 \
-DLLVM_ENABLE_RTTI=ON \
+ -DLLVM_TOOL_LIBCXX_BUILD=OFF \
+ -DLLVM_TOOL_LIBCXXABI_BUILD=OFF \
-DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS" \
-DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
$LLVM_DIR
@@ -117,6 +119,17 @@ build_libstdcxx() {
$GCC_DIR/libstdc++-v3/configure \
--enable-multilib=no \
--prefix="$PREFIX"
+
+ # On Ubuntu distros (tested on 14.04 and 16.04), the configure script has a
+ # nasty habit of disabling TLS support when -fsanitize=thread is used. This
+ # appears to be an interaction between TSAN and the GCC_CHECK_TLS m4 macro
+ # used by configure. It doesn't manifest on el6 because the devtoolset
+ # causes an early conftest to fail, which passes the macro's smell test.
+ #
+ # This is a silly hack to force TLS support back on, but it's only temporary,
+ # as we're about to replace all of this with libc++.
+ sed -ie 's|/\* #undef HAVE_TLS \*/|#define HAVE_TLS 1|' config.h
+
make -j$PARALLEL install
}
http://git-wip-us.apache.org/repos/asf/kudu/blob/dfe96efd/thirdparty/download-thirdparty.sh
----------------------------------------------------------------------
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 7265c1a..191e14a 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -207,14 +207,13 @@ if [ ! -d $PYTHON_DIR ]; then
fetch_and_expand python-${PYTHON_VERSION}.tar.gz
fi
-LLVM_PATCHLEVEL=2
+LLVM_PATCHLEVEL=1
delete_if_wrong_patchlevel $LLVM_DIR $LLVM_PATCHLEVEL
if [ ! -d $LLVM_DIR ]; then
fetch_and_expand llvm-${LLVM_VERSION}.src.tar.gz
pushd $LLVM_DIR
patch -p1 < $TP_DIR/patches/llvm-fix-amazon-linux.patch
- patch -p1 < $TP_DIR/patches/llvm-devtoolset-toolchain.patch
touch patchlevel-$LLVM_PATCHLEVEL
popd
echo
http://git-wip-us.apache.org/repos/asf/kudu/blob/dfe96efd/thirdparty/patches/llvm-devtoolset-toolchain.patch
----------------------------------------------------------------------
diff --git a/thirdparty/patches/llvm-devtoolset-toolchain.patch b/thirdparty/patches/llvm-devtoolset-toolchain.patch
deleted file mode 100644
index ce41be3..0000000
--- a/thirdparty/patches/llvm-devtoolset-toolchain.patch
+++ /dev/null
@@ -1,16 +0,0 @@
-diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
-index 4688335..9173714 100644
---- a/tools/clang/lib/Driver/ToolChains.cpp
-+++ b/tools/clang/lib/Driver/ToolChains.cpp
-@@ -1060,8 +1060,10 @@ Generic_GCC::GCCInstallationDetector::GCCInstallationDetector(
- Prefixes.push_back(D.InstalledDir + "/..");
-
- // And finally in /usr.
-- if (D.SysRoot.empty())
-+ if (D.SysRoot.empty()) {
-+ Prefixes.push_back("/opt/rh/devtoolset-3/root/usr");
- Prefixes.push_back("/usr");
-+ }
- }
-
- // Loop over the various components which exist and select the best GCC
http://git-wip-us.apache.org/repos/asf/kudu/blob/dfe96efd/thirdparty/vars.sh
----------------------------------------------------------------------
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index a477abb..179646c 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -111,13 +111,15 @@ LIBUNWIND_DIR=$TP_DIR/libunwind-${LIBUNWIND_VERSION}
#
# Summary:
# 1. Unpack the llvm tarball
-# 2. Unpack the clang tarball into the llvm tree as tools/clang (rename from cfe-<version> to clang)
-# 3. Unpack the extra clang tools tarball as tools/clang/tools/clang-tools-extra
-# 4. Unpack the lld tarball into the llvm tree as tools/lld
+# 2. Unpack the clang tarball as tools/clang (rename from cfe-<version> to clang)
+# 3. Unpack the extra clang tools tarball as tools/clang/tools/extra
+# 4. Unpack the lld tarball as tools/lld
# 5. Unpack the compiler-rt tarball as projects/compiler-rt
-# 6. Create new tarball from the resulting source tree
+# 6. Unpack the libc++ tarball as projects/libcxx
+# 7. Unpack the libc++abi tarball as projects/libcxxabi
+# 8. Create new tarball from the resulting source tree
#
-LLVM_VERSION=3.8.0
+LLVM_VERSION=3.9.0
LLVM_DIR=$TP_DIR/llvm-${LLVM_VERSION}.src
LLVM_BUILD_DIR=$TP_DIR/llvm-${LLVM_VERSION}.build