You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2022/01/07 01:46:42 UTC
[impala] branch master updated: IMPALA-10951 (preparation): Update Kudu to a more recent version
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 7b490ee IMPALA-10951 (preparation): Update Kudu to a more recent version
7b490ee is described below
commit 7b490eed5b4e18be1ab19e190ec9ad14c2a804d1
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sun Dec 26 21:37:58 2021 -0800
IMPALA-10951 (preparation): Update Kudu to a more recent version
As part of moving to a newer protobuf, this updates the Kudu version
to get the fix for KUDU-3334. With this newer Kudu version, Clang
builds hit an error while linking:
lib/libLLVMCodeGen.a(TargetPassConfig.cpp.o):TargetPassConfig.cpp:
function llvm::TargetPassConfig::createRegAllocPass(bool):
error: relocation refers to global symbol "std::call_once<void (&)()>(std::once_flag&, void (&)())::{lambda()#2}::_FUN()",
which is defined in a discarded section
section group signature: "_ZZSt9call_onceIRFvvEJEEvRSt9once_flagOT_DpOT0_ENKUlvE0_clEv"
prevailing definition is from ../../build/debug/security/libsecurity.a(openssl_util.cc.o)
(This is from a newer binutils that will be pursued separately.)
As a hack to get around this error, this adds the calloncehack
shared library. The shared library publicly defines the symbol that
was coming from kudu_client. By linking it ahead of kudu_client, the
linker uses that rather than the one from kudu_client. This fixes
the Clang builds.
The new Kudu also requires a minor change to the flags for tserver
startup.
Testing:
- Ran debug tests and verified calloncehack is not used
- Ran ASAN tests
Change-Id: Ieccbe284f11445e1de792352ebc7c9e1fa2ca0c3
Reviewed-on: http://gerrit.cloudera.org:8080/18129
Reviewed-by: Wenzhe Zhou <wz...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
CMakeLists.txt | 6 +++
be/CMakeLists.txt | 18 ++++++++
be/src/service/impala-server.cc | 9 ++++
bin/bootstrap_toolchain.py | 9 ++--
bin/impala-config.sh | 6 ++-
cmake_modules/FindCallOnceHack.cmake | 48 ++++++++++++++++++++++
.../common/etc/kudu/tserver.conf.tmpl | 1 +
7 files changed, 91 insertions(+), 6 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3baa187..566c884 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -119,6 +119,7 @@ set_dep_root(THRIFT)
set_dep_root(ZLIB)
set_dep_root(CCTZ)
set_dep_root(CURL)
+set_dep_root(CALLONCEHACK)
# The boost-cmake project hasn't been maintained for years. Let's make sure we
# don't accidentally use it if it can be found.
@@ -352,6 +353,11 @@ find_package(KerberosPrograms REQUIRED)
find_package(Curl REQUIRED)
IMPALA_ADD_THIRDPARTY_LIB(curl ${CURL_INCLUDE_DIR} ${CURL_STATIC_LIB} "")
+# find calloncehack headers and libs
+find_package(CallOnceHack REQUIRED)
+IMPALA_ADD_THIRDPARTY_LIB(calloncehack ${CALLONCEHACK_INCLUDE_DIR} ""
+ ${CALLONCEHACK_SHARED_LIB})
+
# Tests that run any security related tests need to link this in to override the
# krb5_realm_override() implementation in krb5.
# See be/src/kudu/security/krb5_realm_override.cc for more information.
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 09006b9..4860200 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -89,6 +89,7 @@ SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-zero-as-null-pointer-constant")
SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-c++17-extensions")
SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-inconsistent-missing-destructor-override")
SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -Wno-return-type-c-linkage")
+SET(CXX_CLANG_FLAGS "${CXX_CLANG_FLAGS} -DCALLONCEHACK")
# For any gcc builds:
# -g: Enable symbols for profiler tools
# -Wno-unused-local-typedefs: Do not warn for local typedefs that are unused.
@@ -553,6 +554,23 @@ set (IMPALA_DEPENDENCIES
cctz
curl)
+# When building with Clang, linking fails because it is trying to
+# use a symbol in kudu_client, but that symbol is discarded. To
+# hack around this error, the calloncehack shared library defines the
+# same symbol publicly. Placing calloncehack ahead of kudu_client
+# causes the linker to use its definition rather than kudu_client's.
+# The underlying issue is some incompatibility when building with
+# Clang while having libraries built with GCC, so this only applies
+# to Clang compilation.
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+ message(STATUS "C++ compiler is Clang, enabling calloncehack")
+ # Put calloncehack at the start of the dependencies. The important thing
+ # is that it is ahead of kudu_client.
+ set(IMPALA_DEPENDENCIES calloncehack ${IMPALA_DEPENDENCIES})
+else()
+ message(STATUS "C++ compiler is not Clang, skipping calloncehack")
+endif()
+
# Add all external dependencies. They should come after the impala libs.
set (IMPALA_LINK_LIBS ${IMPALA_LINK_LIBS}
${IMPALA_DEPENDENCIES}
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 405a72c..95152f1 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -23,6 +23,9 @@
#include <exception>
#include <fstream>
#include <sstream>
+#ifdef CALLONCEHACK
+#include <calloncehack.h>
+#endif
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/join.hpp>
@@ -2881,6 +2884,12 @@ Status ImpalaServer::Start(int32_t beeswax_port, int32_t hs2_port,
int32_t hs2_http_port, int32_t external_fe_port) {
exec_env_->SetImpalaServer(this);
+#ifdef CALLONCEHACK
+ // Include this calloncehack call (which is a no-op) to make sure calloncehack
+ // is required at link time when using it.
+ calloncehack::InitializeCallOnceHack();
+#endif
+
// We must register the HTTP handlers after registering the ImpalaServer with the
// ExecEnv. Otherwise the HTTP handlers will try to resolve the ImpalaServer through the
// ExecEnv singleton and will receive a nullptr.
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 285686a..bedb9c2 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -463,10 +463,11 @@ def get_toolchain_downloads():
gcc_package = ToolchainPackage("gcc")
toolchain_packages += [llvm_package, llvm_package_asserts, gcc_package]
toolchain_packages += map(ToolchainPackage,
- ["avro", "binutils", "boost", "breakpad", "bzip2", "cctz", "cmake", "crcutil",
- "curl", "flatbuffers", "gdb", "gflags", "glog", "gperftools", "gtest", "jwt-cpp",
- "libev", "libunwind", "lz4", "openldap", "openssl", "orc", "protobuf", "python",
- "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib", "zstd"])
+ ["avro", "binutils", "boost", "breakpad", "bzip2", "calloncehack", "cctz", "cmake",
+ "crcutil", "curl", "flatbuffers", "gdb", "gflags", "glog", "gperftools", "gtest",
+ "jwt-cpp", "libev", "libunwind", "lz4", "openldap", "openssl", "orc", "protobuf",
+ "python", "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib",
+ "zstd"])
# Check whether this platform is supported (or whether a valid custom toolchain
# has been provided).
if not try_get_platform_release_label() \
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index a414864..d34f3a2 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -68,7 +68,7 @@ fi
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=62-4fde3e799d
+export IMPALA_TOOLCHAIN_BUILD_ID=70-b2f92bd26e
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p5
@@ -161,6 +161,8 @@ export IMPALA_THRIFT_VERSION=0.11.0-p4
unset IMPALA_THRIFT_URL
export IMPALA_ZLIB_VERSION=1.2.11
unset IMPALA_ZLIB_URL
+export IMPALA_CALLONCEHACK_VERSION=1.0.0
+unset IMPALA_CALLONCEHACK_URL
if [[ $OSTYPE == "darwin"* ]]; then
IMPALA_CYRUS_SASL_VERSION=2.1.26
@@ -727,7 +729,7 @@ fi
# overall build type) and does not apply when using a local Kudu build.
export USE_KUDU_DEBUG_BUILD=${USE_KUDU_DEBUG_BUILD-false}
-export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"f486f0813a"}
+export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"1754f517b2"}
export IMPALA_KUDU_HOME=${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$IMPALA_KUDU_VERSION
export IMPALA_KUDU_JAVA_HOME=\
${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-${IMPALA_KUDU_VERSION}/java
diff --git a/cmake_modules/FindCallOnceHack.cmake b/cmake_modules/FindCallOnceHack.cmake
new file mode 100644
index 0000000..8c52d84
--- /dev/null
+++ b/cmake_modules/FindCallOnceHack.cmake
@@ -0,0 +1,48 @@
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+
+# CALLONCEHACK_ROOT hints the location
+# Provides
+# - CALLONCEHACK_SHARED_LIB,
+# - CALLONCEHACK_INCLUDE_DIR,
+# - CALLONCEHACK_FOUND
+set(_CALLONCEHACK_SEARCH_DIR)
+
+if (CALLONCEHACK_ROOT)
+set(_CALLONCEHACK_SEARCH_DIR PATHS ${CALLONCEHACK_ROOT} NO_DEFAULT_PATH)
+endif()
+
+find_path(CALLONCEHACK_INCLUDE_DIR calloncehack.h
+ ${_CALLONCEHACK_SEARCH_DIR} PATH_SUFFIXES include)
+
+# Calloncehack is always a shared library
+find_library(CALLONCEHACK_SHARED_LIB libcalloncehack.so ${_CALLONCEHACK_SEARCH_DIR} PATH_SUFFIXES lib)
+
+if (CALLONCEHACK_INCLUDE_DIR AND CALLONCEHACK_SHARED_LIB)
+ set(CALLONCEHACK_FOUND TRUE)
+ message(STATUS "Found the calloncehack library: ${CALLONCEHACK_SHARED_LIB}")
+else()
+ set(CALLONCEHACK_FOUND FALSE)
+ message(FATAL_ERROR "calloncehack not found in ${CALLONCEHACK_ROOT}")
+endif()
+
+mark_as_advanced(
+ CALLONCEHACK_INCLUDE_DIR
+ CALLONCEHACK_SHARED_LIB
+)
diff --git a/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl b/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl
index 71d3f85..810a710 100644
--- a/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl
+++ b/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl
@@ -19,3 +19,4 @@
# Enable Kudu transaction.
-enable_txn_system_client_init
+-unlock_experimental_flags
\ No newline at end of file