You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/09/01 23:10:03 UTC
(impala) 02/02: IMPALA-14759: Upgrade to Boost 1.91
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7a4fc9a9256ed4b90bf275bc18b81318c6bd6bbe
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Thu Feb 19 13:55:07 2026 -0800
IMPALA-14759: Upgrade to Boost 1.91
This adopts Boost 1.91 to have access to a new / fast hash table
and other improvements. There are various small code changes
required to adapt to this change:
- Boost rudely defines INT128_MAX and related symbols in a way
that conflicts with be/src/kudu/util/int128.h. We need to
unset those variables in be/src/runtime/multi-precision.h
to avoid those conflicts.
- Boost moved UUID to use std::mt19937 rather than the boost
version.
- Boost filesystem dropped the is_regular() API in favor
of is_regular_file().
- Boost started embedding a GDB pretty-printer script in
binaries. For codegen, that bloats the codegen size and
provides not benefit, so this turns that off by specifying
-DBOOST_ALL_NO_EMBEDDED_GDB_SCRIPTS.
- Boost changed certain filesystem APIs to use const references,
so a couple locations needed changes to fix clang-tidy issues.
- Boost added a dependency on libquadmath on x86_64, which is
provided by GCC. This updates the x86_64 packaging code to
include libquadmath from our version of GCC. libquadmath does
not support ARM, so it isn't needed there.
This bumps the version of Avro C++ as part of the toolchain
update. Building against Avro C++ is not currently supported,
so this has no impact.
Testing:
- Ran a core job
Change-Id: I4d6ef3e4793faac9b6ca9b4730671dca4bf99da5
Reviewed-on: http://gerrit.cloudera.org:8080/24737
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Michael Smith <mi...@cloudera.com>
---
CMakeLists.txt | 2 +-
be/CMakeLists.txt | 4 +++-
be/src/exec/tuple-file-read-write-test.cc | 2 +-
be/src/rpc/authentication.cc | 6 +++---
be/src/runtime/io/data-cache-test.cc | 2 +-
be/src/runtime/io/scan-range.cc | 1 -
be/src/runtime/multi-precision.h | 5 +++++
be/src/util/uid-util.cc | 2 +-
bin/impala-config.sh | 10 +++++-----
docker/setup_build_context.py | 20 ++++++++++++++++++++
package/CMakeLists.txt | 9 +++++++++
11 files changed, 49 insertions(+), 14 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0a5ce2d97..5b8d1e066 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -212,7 +212,7 @@ if (CMAKE_DEBUG)
endif()
find_package(Boost CONFIG REQUIRED
- COMPONENTS thread regex filesystem system date_time random locale serialization
+ COMPONENTS thread regex filesystem date_time random locale serialization
NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
# Mark Boost as a system header to avoid compile warnings.
include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 412580f21..0b47c145c 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -348,10 +348,12 @@ add_definitions(-DKUDU_HEADERS_USE_RICH_SLICE -DKUDU_HEADERS_NO_STUBS)
# -Wno-return-type-c-linkage: UDFs return C++ classes but use C linkage to prevent
# mangling
# -DBOOST_NO_EXCEPTIONS: call a custom error handler for exceptions in codegen'd code.
+# -DBOOST_ALL_NO_EMBEDDED_GDB_SCRIPTS: don't embed gdb pretty-printer scripts for Boost
+# types. They bloat codegen sizes without providing any value.
set(CLANG_IR_CXX_FLAGS "-emit-llvm" "-c" "-std=c++17" "-DIR_COMPILE" "-DHAVE_INTTYPES_H"
"-DHAVE_NETINET_IN_H" "-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG" "-DBOOST_NO_EXCEPTIONS"
"-DBOOST_BIND_GLOBAL_PLACEHOLDERS" "-DBOOST_ALLOW_DEPRECATED_HEADERS"
- "-DKUDU_HEADERS_NO_STUBS" "-fcolor-diagnostics"
+ "-DBOOST_ALL_NO_EMBEDDED_GDB_SCRIPTS" "-DKUDU_HEADERS_NO_STUBS" "-fcolor-diagnostics"
"-Wno-return-type-c-linkage" "-fsigned-char")
# -Wno-deprecated-declarations: OpenSSL3 deprecated various APIs currently used by
# Impala, so this disables those warnings when using OpenSSL3 until they can be
diff --git a/be/src/exec/tuple-file-read-write-test.cc b/be/src/exec/tuple-file-read-write-test.cc
index f633bf718..66601c7e1 100644
--- a/be/src/exec/tuple-file-read-write-test.cc
+++ b/be/src/exec/tuple-file-read-write-test.cc
@@ -76,7 +76,7 @@ public:
ASSERT_OK(FileSystemUtil::RemovePaths({tmp_dir_.string()}));
}
- string Path(string filename) const {
+ string Path(const string& filename) const {
return (tmp_dir_ / filename).string();
}
diff --git a/be/src/rpc/authentication.cc b/be/src/rpc/authentication.cc
index 691a6cddd..c7ec77787 100644
--- a/be/src/rpc/authentication.cc
+++ b/be/src/rpc/authentication.cc
@@ -77,7 +77,7 @@ using boost::mt19937;
using boost::uniform_int;
using namespace apache::thrift;
using namespace apache::thrift::transport;
-using namespace boost::filesystem; // for is_regular(), is_absolute()
+using namespace boost::filesystem; // for is_regular_file(), is_absolute()
using namespace strings;
DECLARE_bool(skip_external_kerberos_auth);
@@ -1365,7 +1365,7 @@ Status AuthManager::InitKerberosEnv() {
}
if (!FLAGS_keytab_file.empty()) {
- if (!is_regular(FLAGS_keytab_file)) {
+ if (!is_regular_file(FLAGS_keytab_file)) {
return Status(Substitute("Bad --keytab_file value: The file $0 is not a "
"regular file", FLAGS_keytab_file));
}
@@ -1396,7 +1396,7 @@ Status AuthManager::InitKerberosEnv() {
// JAVA_TOOL_OPTIONS in the environment.
if (!FLAGS_krb5_conf.empty()) {
// Ensure it points to a regular file
- if (!is_regular(FLAGS_krb5_conf)) {
+ if (!is_regular_file(FLAGS_krb5_conf)) {
return Status(Substitute("Bad --krb5_conf value: The file $0 is not a "
"regular file", FLAGS_krb5_conf));
}
diff --git a/be/src/runtime/io/data-cache-test.cc b/be/src/runtime/io/data-cache-test.cc
index aa0b6115e..783b0f2f6 100644
--- a/be/src/runtime/io/data-cache-test.cc
+++ b/be/src/runtime/io/data-cache-test.cc
@@ -178,7 +178,7 @@ class DataCacheBaseTest : public testing::Test {
ASSERT_OK(cache->CloseFilesAndVerifySizes());
}
- void MockDisk(std::string name, bool is_rotational) {
+ void MockDisk(const std::string& name, bool is_rotational) {
mock_fs_ = "/tmp" / boost::filesystem::unique_path();
ASSERT_OK(FileSystemUtil::RemoveAndCreateDirectory(mock_fs_.string()));
boost::filesystem::path proc_fs = mock_fs_ / "proc";
diff --git a/be/src/runtime/io/scan-range.cc b/be/src/runtime/io/scan-range.cc
index 463aa9c5b..374598fb3 100644
--- a/be/src/runtime/io/scan-range.cc
+++ b/be/src/runtime/io/scan-range.cc
@@ -27,7 +27,6 @@
using namespace impala;
using namespace impala::io;
-using namespace std;
using std::mutex;
using std::unique_lock;
diff --git a/be/src/runtime/multi-precision.h b/be/src/runtime/multi-precision.h
index 485355321..d2a09c319 100644
--- a/be/src/runtime/multi-precision.h
+++ b/be/src/runtime/multi-precision.h
@@ -44,6 +44,11 @@
#else
#include <boost/multiprecision/cpp_int.hpp>
#include <boost/multiprecision/cpp_dec_float.hpp>
+// Boost multiprecision defines these macros, but these interfere with kudu/util/int128.h
+#undef INT128_MIN
+#undef INT128_MAX
+#undef UINT128_MIN
+#undef UINT128_MAX
#endif
#include <functional>
diff --git a/be/src/util/uid-util.cc b/be/src/util/uid-util.cc
index 9910cc4c9..a886b971e 100644
--- a/be/src/util/uid-util.cc
+++ b/be/src/util/uid-util.cc
@@ -24,7 +24,7 @@
namespace impala {
string GenerateUUIDString() {
- boost::uuids::basic_random_generator<boost::mt19937> gen;
+ boost::uuids::basic_random_generator<std::mt19937> gen;
boost::uuids::uuid u = gen();
string uuid(u.begin(), u.end());
return uuid;
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index f92c67a67..9338683af 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -81,13 +81,13 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false}
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=189-f0dd1e02a0
-export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=663-f0dd1e02a0
+export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=199-30fcdeb21b
+export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=672-30fcdeb21b
export IMPALA_TOOLCHAIN_REPO=\
${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git}
export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master}
export IMPALA_TOOLCHAIN_COMMIT_HASH=\
-${IMPALA_TOOLCHAIN_COMMIT_HASH-f0dd1e02a081aae0cec7669f5fdadc9d54a5d259}
+${IMPALA_TOOLCHAIN_COMMIT_HASH-30fcdeb21b0d73534a2e9d4c4328a9a7a3a3d52f}
# Compare the build ref in build IDs by removing everything up-to-and-including the
# first hyphen.
if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
@@ -110,7 +110,7 @@ export ARCH_NAME=$(uname -m)
# Versions of toolchain dependencies.
# -----------------------------------
if $USE_AVRO_CPP; then
- export IMPALA_AVRO_VERSION=1.11.1-p1
+ export IMPALA_AVRO_VERSION=1.12.2-p1
else
export IMPALA_AVRO_VERSION=1.7.4-p5
fi
@@ -119,7 +119,7 @@ export IMPALA_ABSEIL_CPP_VERSION=20250512.2
unset IMPALA_ABSEIL_CPP_URL
export IMPALA_BINUTILS_VERSION=2.42-p24
unset IMPALA_BINUTILS_URL
-export IMPALA_BOOST_VERSION=1.74.0-p1
+export IMPALA_BOOST_VERSION=1.91.0-p1
unset IMPALA_BOOST_URL
export IMPALA_BREAKPAD_VERSION=e09741c609dcd5f5274d40182c5e2cc9a002d5ba-p3
unset IMPALA_BREAKPAD_URL
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index f5e51ee97..b2ff74f93 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -22,6 +22,7 @@
import argparse
import glob
+import platform
import os
import shutil
from subprocess import check_call
@@ -162,6 +163,25 @@ if not found_libgcc_so:
raise Exception("No libgcc.so found in search path: {0}".format(
os.path.join(GCC_HOME, "lib64")))
+# libquadmath is needed for Boost charconv / locale on x86_64 (but not ARM)
+# platform.machine() is based on "uname -m".
+if platform.machine() == "x86_64":
+ # Add libquadmath binaries to LIB_DIR (needed for Boost charconv / locale)
+ found_libquadmath_so = False
+ for libquadmath_so in glob.glob(os.path.join(
+ GCC_HOME, "lib64/{0}*.so*".format("libquadmath"))):
+ found_libquadmath_so = True
+ symlink_file_into_dirs(libquadmath_so, TARGET_LIB_DIRS)
+
+ if not found_libquadmath_so:
+ raise Exception("No libquadmath.so found in search path: {0}".format(
+ os.path.join(GCC_HOME, "lib64")))
+elif platform.machine() == "aarch64":
+ if len(glob.glob(os.path.join(GCC_HOME, "lib64/{0}*.so*".format("libquadmath")))) != 0:
+ raise Exception("Found libquadmath.so, but it is not expected to exist on ARM64")
+else:
+ raise Exception("Unknown platform {0}".format(platform.machine()))
+
# Add libkudu_client binaries to LIB_DIR. Strip debug symbols for release builds.
found_kudu_so = False
for kudu_lib_dir in kudu_lib_dirs:
diff --git a/package/CMakeLists.txt b/package/CMakeLists.txt
index 8e4c4273f..afb37525a 100644
--- a/package/CMakeLists.txt
+++ b/package/CMakeLists.txt
@@ -42,6 +42,15 @@ if("${cpp_lib}" STREQUAL "")
endif()
install(FILES ${cpp_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib/native)
+# Boost requires libquadmath on x86_64 (libquadmath is not supported on ARM)
+if ("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "x86_64")
+ FILE(GLOB quadmath_lib ${IMPALA_GCC_HOME}/lib64/libquadmath.so*)
+ if("${quadmath_lib}" STREQUAL "")
+ message(FATAL_ERROR "libquadmath.so was not found in '${IMPALA_GCC_HOME}/lib64'!")
+ endif()
+ install(FILES ${quadmath_lib} DESTINATION ${IMPALA_INSTALLDIR}/lib/native)
+endif()
+
set(KUDU_HOME $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$ENV{IMPALA_KUDU_VERSION}/release)
# The parent folder is lib64 on centos/redhat, while on ubuntu it's lib.
FILE(GLOB kudu_lib ${KUDU_HOME}/lib*/libkudu_client.so*)