You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/24 04:41:06 UTC
[doris] branch dev-1.0.1 updated: [hotfix] cherry pick thirdparty updates to dev-1.0.1
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/dev-1.0.1 by this push:
new f4f134b9b9 [hotfix] cherry pick thirdparty updates to dev-1.0.1
f4f134b9b9 is described below
commit f4f134b9b992c078bde721e15ba6c062ed824eac
Author: morningman <mo...@163.com>
AuthorDate: Fri Jun 24 12:39:19 2022 +0800
[hotfix] cherry pick thirdparty updates to dev-1.0.1
Make the thirdparty build well.
---
thirdparty/CHANGELOG.md | 28 +-
thirdparty/build-thirdparty.sh | 123 ++++++--
thirdparty/download-thirdparty.sh | 30 +-
thirdparty/patches/hyperscan-5.4.0.patch | 18 ++
thirdparty/patches/opentelemetry-cpp-1.4.0.patch | 374 +++++++++++++++++++++++
thirdparty/vars.sh | 51 +++-
6 files changed, 590 insertions(+), 34 deletions(-)
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index 17742e549b..4a936be5fc 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -1,9 +1,33 @@
# Changelog
-This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/incubator-doris, and the tag is `build-env-${version}`
+This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
+
+## v20220613
+- Modified: update libhdfs3 from 2.3.0 to 2.3.1 fix client uuid set error
+
+## v20220608
+- Remove: remove libhdfs3 without kerberos support
+- Modified: make libhdfs3 with kerberos support as default
+- Modified: change libhdfs3 to https://github.com/yangzhg/libhdfs3/releases/tag/v2.3.0 . This version support arm CPUs
+
+## v20220607
+- Added: opentelemetry-cpp 1.4.0, it was introduced for tracing.
+- Added: opentelemetry-proto 0.18.0, it is depended on by opentelemetry-cpp.
+- Added: nlohmann/json 3.10.1, it is depended on by opentelemetry-cpp.
+
+## v20220606
+- Added: hyperscan 5.4.0, and a patch for compilation
+- Added: ragel 6.1.0, it is used by hyperscan to generate files before compilation
+
+## v20220522
+
+- Added: libgsasl 1.8.0, this version of gsasl is only used for libhdfs3 with kerberos
+- Added: krb5 1.19
+
+Now there will be 2 set of libhdfs, one is without kerberos, the other is with kerberos, saved in `thirdparty/installed/libhdfs_with_kerberos/`
## v20220321
-- Add libbacktrace, it is used by boost stacktrace to print exception stack.
+- Added: libbacktrace, it is used by boost stacktrace to print exception stack.
## v20220316
- Modified: CRoaring 0.3.4 -> 0.4.0
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index d7f412cfd9..fa92259c21 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -311,9 +311,19 @@ build_protobuf() {
CXXFLAGS="-fPIC -O2 -I${TP_INCLUDE_DIR}" \
LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc -Wl,--undefined=pthread_create" \
./configure --prefix=${TP_INSTALL_DIR} --disable-shared --enable-static --with-zlib=${TP_INSTALL_DIR}/include
- cd src
- sed -i 's/^AM_LDFLAGS\(.*\)$/AM_LDFLAGS\1 -all-static/' Makefile
- cd -
+
+ # ATTN: If protoc is not built fully statically the linktime libc may newer than runtime.
+ # This will casue protoc cannot run
+ # If you really need to dynamically link protoc, please set the environment variable DYN_LINK_PROTOC=1
+
+ if [[ "${DYN_LINK_PROTOC}" == "1" ]]; then
+ echo "link protoc dynamiclly"
+ else
+ cd src
+ sed -i 's/^AM_LDFLAGS\(.*\)$/AM_LDFLAGS\1 -all-static/' Makefile
+ cd -
+ fi
+
make -j $PARALLEL && make install
}
@@ -360,9 +370,12 @@ build_gtest() {
# rapidjson
build_rapidjson() {
check_if_source_exist $RAPIDJSON_SOURCE
-
- rm -rf $TP_INSTALL_DIR/rapidjson
- cp -r $TP_SOURCE_DIR/$RAPIDJSON_SOURCE/include/rapidjson $TP_INCLUDE_DIR/
+ cd $TP_SOURCE_DIR/$RAPIDJSON_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ rm -rf CMakeCache.txt CMakeFiles/
+ ${CMAKE_CMD} ../ -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DRAPIDJSON_BUILD_DOC=OFF \
+ -DRAPIDJSON_BUILD_EXAMPLES=OFF -DRAPIDJSON_BUILD_TESTS=OFF
+ make -j $PARALLEL && make install
}
# snappy
@@ -373,7 +386,7 @@ build_snappy() {
mkdir -p $BUILD_DIR && cd $BUILD_DIR
rm -rf CMakeCache.txt CMakeFiles/
CFLAGS="-O3" CXXFLAGS="-O3" ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
- -DCMAKE_POSITION_INDEPENDENT_CODE=On \
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_INSTALL_INCLUDEDIR=$TP_INCLUDE_DIR/snappy \
-DSNAPPY_BUILD_TESTS=0 ../
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
@@ -492,6 +505,25 @@ build_re2() {
${BUILD_SYSTEM} -j $PARALLEL install
}
+# hyperscan
+build_hyperscan() {
+ MACHINE_TYPE=$(uname -m)
+ if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then
+ echo "hyperscan is not supporting aarch64 now."
+ else
+ check_if_source_exist $RAGEL_SOURCE
+ cd $TP_SOURCE_DIR/$RAGEL_SOURCE
+ ./configure --prefix=$TP_INSTALL_DIR && make install
+
+ check_if_source_exist $HYPERSCAN_SOURCE
+ cd $TP_SOURCE_DIR/$HYPERSCAN_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ PATH=$TP_INSTALL_DIR/bin:$PATH ${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 \
+ -DBOOST_ROOT=$BOOST_SOURCE -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR ..
+ ${BUILD_SYSTEM} -j $PARALLEL install
+ fi
+}
+
# boost
build_boost() {
check_if_source_exist $BOOST_SOURCE
@@ -522,7 +554,7 @@ build_mysql() {
-DWITHOUT_SERVER=1 -DWITH_ZLIB=1 -DZLIB_ROOT=$TP_INSTALL_DIR \
-DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O3 -g -fabi-version=2 -fno-omit-frame-pointer -fno-strict-aliasing -std=gnu++11" \
-DDISABLE_SHARED=1 -DBUILD_SHARED_LIBS=0 -DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a -DENABLE_DTRACE=0
- ${BUILD_SYSTEM} -v -j $PARALLEL mysqlclient
+ ${BUILD_SYSTEM} -j $PARALLEL mysqlclient
# copy headers manually
rm -rf ../../../installed/include/mysql/
@@ -643,20 +675,24 @@ build_arrow() {
export ARROW_SNAPPY_URL=${TP_SOURCE_DIR}/${SNAPPY_NAME}
export ARROW_ZLIB_URL=${TP_SOURCE_DIR}/${ZLIB_NAME}
export ARROW_XSIMD_URL=${TP_SOURCE_DIR}/${XSIMD_NAME}
+ export ARROW_ORC_URL=${TP_SOURCE_DIR}/${ORC_NAME}
LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
${CMAKE_CMD} -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_BUILD_SHARED=OFF \
-DARROW_BUILD_STATIC=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON -DARROW_USE_GLOG=ON \
-DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_JSON=ON \
- -DARROW_WITH_UTF8PROC=OFF -DARROW_WITH_RE2=OFF \
+ -DARROW_WITH_UTF8PROC=OFF -DARROW_WITH_RE2=ON -DARROW_ORC=ON \
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DARROW_BOOST_USE_SHARED=OFF \
+ -DBoost_USE_STATIC_RUNTIME=ON \
-DARROW_GFLAGS_USE_SHARED=OFF \
-Dgflags_ROOT=$TP_INSTALL_DIR \
-DGLOG_ROOT=$TP_INSTALL_DIR \
+ -DRE2_ROOT=$TP_INSTALL_DIR \
-DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a -DZLIB_INCLUDE_DIR=$TP_INSTALL_DIR/include \
-DRapidJSON_ROOT=$TP_INSTALL_DIR \
+ -DORC_ROOT=$TP_INSTALL_DIR \
-DBrotli_SOURCE=BUNDLED \
-DLZ4_LIB=$TP_INSTALL_DIR/lib/liblz4.a -DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \
-DLz4_SOURCE=SYSTEM \
@@ -664,7 +700,7 @@ build_arrow() {
-Dzstd_SOURCE=SYSTEM \
-DSnappy_LIB=$TP_INSTALL_DIR/lib/libsnappy.a -DSnappy_INCLUDE_DIR=$TP_INSTALL_DIR/include \
-DSnappy_SOURCE=SYSTEM \
- -DBoost_INCLUDE_DIR=$TP_INSTALL_DIR/include \
+ -DBOOST_ROOT=$TP_INSTALL_DIR --no-warn-unused-cli \
-DThrift_ROOT=$TP_INSTALL_DIR ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
@@ -749,13 +785,19 @@ build_bitshuffle() {
# croaring bitmap
build_croaringbitmap() {
+ avx_flag=
+ if [ ! -z "$USE_AVX2" -a "$USE_AVX2" -eq 0 ];then
+ echo "set USE_AVX2=$USE_AVX2 to FORCE disable AVX2 in croaringbitmap"
+ avx_flag="-DROARING_DISABLE_AVX=ON"
+ fi
+
check_if_source_exist $CROARINGBITMAP_SOURCE
cd $TP_SOURCE_DIR/$CROARINGBITMAP_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
rm -rf CMakeCache.txt CMakeFiles/
CXXFLAGS="-O3" \
LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
- ${CMAKE_CMD} -G "${GENERATOR}" -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
+ ${CMAKE_CMD} -G "${GENERATOR}" ${avx_flag} -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DENABLE_ROARING_TESTS=OFF ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -863,6 +905,10 @@ build_aws_sdk() {
# lzma
build_lzma() {
+ if [ ! -x "$(command -v autopoint)" ]; then
+ echo "autopoint is required by $0, install it first"
+ return -1
+ fi
check_if_source_exist $LZMA_SOURCE
cd $TP_SOURCE_DIR/$LZMA_SOURCE
export ACLOCAL_PATH=/usr/share/aclocal
@@ -874,6 +920,10 @@ build_lzma() {
# xml2
build_xml2() {
+ if [ ! -x "$(command -v pkg-config)" ]; then
+ echo "pkg-config is required by $0, install it first"
+ return -1
+ fi
check_if_source_exist $XML2_SOURCE
cd $TP_SOURCE_DIR/$XML2_SOURCE
export ACLOCAL_PATH=/usr/share/aclocal
@@ -919,24 +969,13 @@ build_hdfs3() {
check_if_source_exist $HDFS3_SOURCE
cd $TP_SOURCE_DIR/$HDFS3_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
- # build libhdfs3 without kerberos
+ # build libhdfs3 with kerberos support
CPPLAGS="-I${TP_INCLUDE_DIR} -fPIC" \
LDFLAGS="-L${TP_LIB_DIR}" \
../bootstrap --dependency="$TP_INSTALL_DIR" --prefix=$TP_INSTALL_DIR --disable-shared --enable-static
make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
make install
}
-
-# hdfs3_with_kerberos
-build_hdfs3_with_kerberos() {
- check_if_source_exist $HDFS3_SOURCE
- cd $TP_SOURCE_DIR/$HDFS3_SOURCE
- mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
- # build libhdfs3 with kerberos support
- ../bootstrap --dependency="$HDFS3_KRB5_INSTALL_DIR:$TP_INSTALL_DIR -DWITH_KERBEROS=true" --prefix=$HDFS3_KRB5_INSTALL_DIR
- make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
- make install
-}
# benchmark
build_benchmark() {
check_if_source_exist $BENCHMARK_SOURCE
@@ -960,8 +999,8 @@ build_simdjson() {
cd $TP_SOURCE_DIR/$SIMDJSON_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- CXX_FLAGS="-O3" \
- C_FLAGS="-O3" \
+ CXXFLAGS="-O3" \
+ CFLAGS="-O3" \
$CMAKE_CMD ..
$CMAKE_CMD --build .
@@ -970,6 +1009,34 @@ build_simdjson() {
cp -r $TP_SOURCE_DIR/$SIMDJSON_SOURCE/include/* $TP_INCLUDE_DIR/
}
+# nlohmann_json
+build_nlohmann_json() {
+ check_if_source_exist $NLOHMANN_JSON_SOURCE
+ cd $TP_SOURCE_DIR/$NLOHMANN_JSON_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+
+ $CMAKE_CMD -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DJSON_BuildTests=OFF ..
+ ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
+}
+
+# opentelemetry
+build_opentelemetry() {
+ check_if_source_exist $OPENTELEMETRY_SOURCE
+ cd $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+
+ $CMAKE_CMD -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_TESTING=OFF \
+ -DWITH_OTLP=ON -DWITH_OTLP_GRPC=OFF -DWITH_OTLP_HTTP=ON -DWITH_ZIPKIN=ON -DWITH_EXAMPLES=OFF ..
+ ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
+}
+
+# sse2neon
+build_sse2neon() {
+ check_if_source_exist $SSE2NEON_SOURCE
+ cd $TP_SOURCE_DIR/$SSE2NEON_SOURCE
+ cp sse2neon.h $TP_INSTALL_DIR/include/
+}
+
build_libunixodbc
build_openssl
build_libevent
@@ -988,6 +1055,7 @@ build_snappy
build_gperftools
build_curl
build_re2
+# build_hyperscan
build_thrift
build_leveldb
build_brpc
@@ -995,6 +1063,7 @@ build_rocksdb
build_cyrus_sasl
build_librdkafka
build_flatbuffers
+build_orc
build_arrow
build_s2
build_bitshuffle
@@ -1003,7 +1072,6 @@ build_fmt
build_parallel_hashmap
build_pdqsort
build_libdivide
-build_orc
build_cctz
build_tsan_header
build_mysql
@@ -1017,7 +1085,10 @@ build_krb5
build_hdfs3
build_benchmark
build_simdjson
+build_nlohmann_json
+build_opentelemetry
build_libbacktrace
+build_sse2neon
echo "Finished to build all thirdparties"
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 388283aa6f..c9d82949a3 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -271,7 +271,7 @@ fi
cd -
echo "Finished patching $S2_SOURCE"
-# gsasl patch to fix link error such as mutilple func defination
+# gsasl2 patch to fix link error such as mutilple func defination
# when link target with kerberos
cd $TP_SOURCE_DIR/$GSASL_SOURCE
if [ ! -f $PATCHED_MARK ]; then
@@ -292,6 +292,22 @@ if [ $ROCKSDB_SOURCE == "rocksdb-5.14.2" ]; then
fi
echo "Finished patching $ROCKSDB_SOURCE"
+# opentelemetry patch is used to solve the problem that threadlocal depends on GLIBC_2.18
+# see: https://github.com/apache/doris/pull/7911
+if [ $OPENTELEMETRY_SOURCE == "opentelemetry-cpp-1.4.0" ]; then
+ rm -rf $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/*
+ cp -r $TP_SOURCE_DIR/$OPENTELEMETRY_PROTO_SOURCE/* $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto
+ mkdir -p $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE/third_party/opentelemetry-proto/.git
+
+ cd $TP_SOURCE_DIR/$OPENTELEMETRY_SOURCE
+ if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/opentelemetry-cpp-1.4.0.patch
+ touch $PATCHED_MARK
+ fi
+ cd -
+fi
+echo "Finished patching $OPENTELEMETRY_SOURCE"
+
# patch librdkafka to avoid crash
if [ $LIBRDKAFKA_SOURCE = "librdkafka-1.8.2" ]; then
cd $TP_SOURCE_DIR/$LIBRDKAFKA_SOURCE
@@ -303,6 +319,18 @@ if [ $LIBRDKAFKA_SOURCE = "librdkafka-1.8.2" ]; then
fi
echo "Finished patching $LIBRDKAFKA_SOURCE"
+# patch hyperscan
+# https://github.com/intel/hyperscan/issues/292
+if [ $HYPERSCAN_SOURCE = "hyperscan-5.4.0" ]; then
+ cd $TP_SOURCE_DIR/$HYPERSCAN_SOURCE
+ if [ ! -f $PATCHED_MARK ]; then
+ patch -p0 < $TP_PATCH_DIR/hyperscan-5.4.0.patch
+ touch $PATCHED_MARK
+ fi
+ cd -
+fi
+echo "Finished patching $HYPERSCAN_SOURCE"
+
cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE
if [ ! -f $PATCHED_MARK ]; then
if [ $AWS_SDK_SOURCE == "aws-sdk-cpp-1.9.211" ]; then
diff --git a/thirdparty/patches/hyperscan-5.4.0.patch b/thirdparty/patches/hyperscan-5.4.0.patch
new file mode 100644
index 0000000000..8bce7a88ba
--- /dev/null
+++ b/thirdparty/patches/hyperscan-5.4.0.patch
@@ -0,0 +1,18 @@
+diff --git cmake/build_wrapper.sh cmake/build_wrapper.sh
+index 1962813..becfbf4 100755
+--- cmake/build_wrapper.sh
++++ cmake/build_wrapper.sh
+@@ -17,11 +17,11 @@ KEEPSYMS=$(mktemp -p /tmp keep.syms.XXXXX)
+ LIBC_SO=$("$@" --print-file-name=libc.so.6)
+ cp ${KEEPSYMS_IN} ${KEEPSYMS}
+ # get all symbols from libc and turn them into patterns
+-nm -f p -g -D ${LIBC_SO} | sed -s 's/\([^ ]*\).*/^\1$/' >> ${KEEPSYMS}
++nm -f posix -g -D ${LIBC_SO} | sed -s 's/\([^ @]*\).*/^\1$/' >> ${KEEPSYMS}
+ # build the object
+ "$@"
+ # rename the symbols in the object
+-nm -f p -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE}
++nm -f posix -g ${OUT} | cut -f1 -d' ' | grep -v -f ${KEEPSYMS} | sed -e "s/\(.*\)/\1\ ${PREFIX}_\1/" >> ${SYMSFILE}
+ if test -s ${SYMSFILE}
+ then
+ objcopy --redefine-syms=${SYMSFILE} ${OUT}
diff --git a/thirdparty/patches/opentelemetry-cpp-1.4.0.patch b/thirdparty/patches/opentelemetry-cpp-1.4.0.patch
new file mode 100644
index 0000000000..2862d11c29
--- /dev/null
+++ b/thirdparty/patches/opentelemetry-cpp-1.4.0.patch
@@ -0,0 +1,374 @@
+diff --git a/api/include/opentelemetry/common/threadlocal.h b/api/include/opentelemetry/common/threadlocal.h
+new file mode 100644
+index 0000000..799ec6c
+--- /dev/null
++++ b/api/include/opentelemetry/common/threadlocal.h
+@@ -0,0 +1,206 @@
++// Copyright The OpenTelemetry Authors
++// SPDX-License-Identifier: Apache-2.0
++
++#pragma once
++
++#include <pthread.h>
++
++#include <memory>
++#include <mutex>
++#include <ostream>
++
++//
++// GCC can be told that a certain branch is not likely to be taken (for
++// instance, a CHECK failure), and use that information in static analysis.
++// Giving it this information can help it optimize for the common case in
++// the absence of better information (ie. -fprofile-arcs).
++//
++#ifndef PREDICT_FALSE
++#if defined(__GNUC__)
++#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
++#else
++#define PREDICT_FALSE(x) x
++#endif
++#endif
++#ifndef PREDICT_TRUE
++#if defined(__GNUC__)
++#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
++#else
++#define PREDICT_TRUE(x) x
++#endif
++#endif
++
++// Block-scoped static thread local implementation.
++//
++// Usage is similar to a C++11 thread_local. The BLOCK_STATIC_THREAD_LOCAL_TELEMTRY macro
++// defines a thread-local pointer to the specified type, which is lazily
++// instantiated by any thread entering the block for the first time. The
++// constructor for the type T is invoked at macro execution time, as expected,
++// and its destructor is invoked when the corresponding thread's Runnable
++// returns, or when the thread exits.
++//
++// Inspired by Poco <http://pocoproject.org/docs/Poco.ThreadLocal.html>,
++// Andrew Tomazos <http://stackoverflow.com/questions/12049684/>, and
++// the C++11 thread_local API.
++//
++// Example usage:
++//
++// // Invokes a 3-arg constructor on SomeClass:
++// BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(SomeClass, instance, arg1, arg2, arg3);
++// instance->DoSomething();
++//
++#define BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(T, t, ...) \
++static __thread T* t; \
++do { \
++ if (PREDICT_FALSE(t == NULL)) { \
++ t = new T(__VA_ARGS__); \
++ internal_threadlocal::ThreadLocal::AddDestructor(internal_threadlocal::ThreadLocal::Destroy<T>, t); \
++ } \
++} while (false)
++
++// Class-scoped static thread local implementation.
++//
++// Very similar in implementation to the above block-scoped version, but
++// requires a bit more syntax and vigilance to use properly.
++//
++// DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(Type, instance_var_) must be placed in the
++// class header, as usual for variable declarations.
++//
++// Because these variables are static, they must also be defined in the impl
++// file with DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(Type, Classname, instance_var_),
++// which is very much like defining any static member, i.e. int Foo::member_.
++//
++// Finally, each thread must initialize the instance before using it by calling
++// INIT_STATIC_THREAD_LOCAL_TELEMETRY(Type, instance_var_, ...). This is a cheap
++// call, and may be invoked at the top of any method which may reference a
++// thread-local variable.
++//
++// Due to all of these requirements, you should probably declare TLS members
++// as private.
++//
++// Example usage:
++//
++// // foo.h
++// #include "kudu/utils/file.h"
++// class Foo {
++// public:
++// void DoSomething(std::string s);
++// private:
++// DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(utils::File, file_);
++// };
++//
++// // foo.cc
++// #include "kudu/foo.h"
++// DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(utils::File, Foo, file_);
++// void Foo::WriteToFile(std::string s) {
++// // Call constructor if necessary.
++// INIT_STATIC_THREAD_LOCAL_TELEMETRY(utils::File, file_, "/tmp/file_location.txt");
++// file_->Write(s);
++// }
++
++// Goes in the class declaration (usually in a header file).
++// dtor must be destructed _after_ t, so it gets defined first.
++// Uses a mangled variable name for dtor since it must also be a member of the
++// class.
++#define DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(T, t) \
++static __thread T* t
++
++// You must also define the instance in the .cc file.
++#define DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(T, Class, t) \
++__thread T* Class::t
++
++// Must be invoked at least once by each thread that will access t.
++#define INIT_STATIC_THREAD_LOCAL_TELEMETRY(T, t, ...) \
++do { \
++ if (PREDICT_FALSE(t == NULL)) { \
++ t = new T(__VA_ARGS__); \
++ internal_threadlocal::ThreadLocal::AddDestructor(internal_threadlocal::ThreadLocal::Destroy<T>, t); \
++ } \
++} while (false)
++
++// Internal implementation below.
++OPENTELEMETRY_BEGIN_NAMESPACE
++namespace internal_threadlocal
++{
++
++// One key used by the entire process to attach destructors on thread exit.
++static pthread_key_t destructors_key;
++
++static std::once_flag once_init;
++
++namespace
++{
++// List of destructors for all thread locals instantiated on a given thread.
++struct PerThreadDestructorList
++{
++ void (*destructor)(void *);
++ void *arg;
++ PerThreadDestructorList *next;
++};
++
++} // anonymous namespace
++
++class ThreadLocal {
++public:
++// Destroy the passed object of type T.
++ template <class T>
++ static void Destroy(void *t)
++ {
++ // With tcmalloc, this should be pretty cheap (same thread as new).
++ delete reinterpret_cast<T *>(t);
++ }
++
++ // Call all the destructors associated with all THREAD_LOCAL instances in this
++// thread.
++ static void InvokeDestructors(void *t)
++ {
++ PerThreadDestructorList *d = reinterpret_cast<PerThreadDestructorList *>(t);
++ while (d != nullptr)
++ {
++ d->destructor(d->arg);
++ PerThreadDestructorList *next = d->next;
++ delete d;
++ d = next;
++ }
++ }
++
++// This key must be initialized only once.
++ static void CreateKey()
++ {
++ int ret = pthread_key_create(&destructors_key, &InvokeDestructors);
++ // Linux supports up to 1024 keys, we will use only one for all thread locals.
++/* if (ret != 0)
++ {
++ std::stringstream ss;
++ ss << "[thread local] pthread_key_create() failed, cannot add destructor to thread: "
++ << "error " << ret;
++ OTEL_INTERNAL_LOG_ERROR(ss.str());
++ }*/
++ }
++
++// Adds a destructor to the list.
++ static void AddDestructor(void (*destructor)(void *), void *arg)
++ {
++ std::call_once(once_init, &CreateKey);
++
++ // Returns NULL if nothing is set yet.
++ std::unique_ptr<PerThreadDestructorList> p(new PerThreadDestructorList());
++ p->destructor = destructor;
++ p->arg = arg;
++ p->next = reinterpret_cast<PerThreadDestructorList *>(pthread_getspecific(destructors_key));
++ int ret = pthread_setspecific(destructors_key, p.release());
++ // The only time this check should fail is if we are out of memory, or if
++ // somehow key creation failed, which should be caught by the above CHECK.
++/* if (ret != 0)
++ {
++ std::stringstream ss;
++ ss << "[thread local] pthread_setspecific() failed, cannot update destructor list: "
++ << "error " << ret;
++ OTEL_INTERNAL_LOG_ERROR(ss.str());
++ }*/
++ }
++
++};
++
++} // namespace threadlocal
++OPENTELEMETRY_END_NAMESPACE
+\ No newline at end of file
+diff --git a/api/include/opentelemetry/context/runtime_context.h b/api/include/opentelemetry/context/runtime_context.h
+index 5cb793b..24ba44b 100644
+--- a/api/include/opentelemetry/context/runtime_context.h
++++ b/api/include/opentelemetry/context/runtime_context.h
+@@ -4,6 +4,7 @@
+ #pragma once
+
+ #include "opentelemetry/context/context.h"
++#include "opentelemetry/common/threadlocal.h"
+
+ OPENTELEMETRY_BEGIN_NAMESPACE
+ namespace context
+@@ -178,7 +179,7 @@ class ThreadLocalContextStorage : public RuntimeContextStorage
+ ThreadLocalContextStorage() noexcept = default;
+
+ // Return the current context.
+- Context GetCurrent() noexcept override { return GetStack().Top(); }
++ Context GetCurrent() noexcept override { return GetStack()->Top(); }
+
+ // Resets the context to the value previous to the passed in token. This will
+ // also detach all child contexts of the passed in token.
+@@ -186,23 +187,23 @@ class ThreadLocalContextStorage : public RuntimeContextStorage
+ bool Detach(Token &token) noexcept override
+ {
+ // In most cases, the context to be detached is on the top of the stack.
+- if (token == GetStack().Top())
++ if (token == GetStack()->Top())
+ {
+- GetStack().Pop();
++ GetStack()->Pop();
+ return true;
+ }
+
+- if (!GetStack().Contains(token))
++ if (!GetStack()->Contains(token))
+ {
+ return false;
+ }
+
+- while (!(token == GetStack().Top()))
++ while (!(token == GetStack()->Top()))
+ {
+- GetStack().Pop();
++ GetStack()->Pop();
+ }
+
+- GetStack().Pop();
++ GetStack()->Pop();
+
+ return true;
+ }
+@@ -211,14 +212,14 @@ class ThreadLocalContextStorage : public RuntimeContextStorage
+ // that can be used to reset to the previous Context.
+ nostd::unique_ptr<Token> Attach(const Context &context) noexcept override
+ {
+- GetStack().Push(context);
++ GetStack()->Push(context);
+ return CreateToken(context);
+ }
+
+-private:
+ // A nested class to store the attached contexts in a stack.
+ class Stack
+ {
++ public:
+ friend class ThreadLocalContextStorage;
+
+ Stack() noexcept : size_(0), capacity_(0), base_(nullptr){};
+@@ -305,9 +306,10 @@ class ThreadLocalContextStorage : public RuntimeContextStorage
+ Context *base_;
+ };
+
+- Stack &GetStack()
++ Stack *GetStack()
+ {
+- static thread_local Stack stack_ = Stack();
++ // static thread_local Stack stack_ = Stack();
++ BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(ThreadLocalContextStorage::Stack, stack_);
+ return stack_;
+ }
+ };
+diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt
+index 9aa4588..e2c5a4a 100644
+--- a/sdk/CMakeLists.txt
++++ b/sdk/CMakeLists.txt
+@@ -1,4 +1,4 @@
+-add_library(opentelemetry_sdk INTERFACE)
++add_library(opentelemetry_sdk INTERFACE ../api/include/opentelemetry/common/threadlocal.h)
+ target_include_directories(
+ opentelemetry_sdk
+ INTERFACE "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>"
+diff --git a/sdk/src/common/random.cc b/sdk/src/common/random.cc
+index 77b88cf..dc71f9c 100644
+--- a/sdk/src/common/random.cc
++++ b/sdk/src/common/random.cc
+@@ -3,6 +3,7 @@
+ // SPDX-License-Identifier: Apache-2.0
+
+ #include "src/common/random.h"
++#include "opentelemetry/common/threadlocal.h"
+ #include "src/common/platform/fork.h"
+
+ #include <cstring>
+@@ -29,33 +30,37 @@ class TlsRandomNumberGenerator
+ platform::AtFork(nullptr, nullptr, OnFork);
+ }
+
+- static FastRandomNumberGenerator &engine() noexcept { return engine_; }
++ static FastRandomNumberGenerator *engine() noexcept { return engine_; }
+
+ private:
+- static thread_local FastRandomNumberGenerator engine_;
++ // static thread_local FastRandomNumberGenerator engine_;
++ DECLARE_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, engine_);
+
+ static void OnFork() noexcept { Seed(); }
+
+ static void Seed() noexcept
+ {
++ INIT_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, engine_);
+ std::random_device random_device;
+ std::seed_seq seed_seq{random_device(), random_device(), random_device(), random_device()};
+- engine_.seed(seed_seq);
++ engine_->seed(seed_seq);
+ }
+ };
+
+-thread_local FastRandomNumberGenerator TlsRandomNumberGenerator::engine_{};
++// thread_local FastRandomNumberGenerator TlsRandomNumberGenerator::engine_{};
++DEFINE_STATIC_THREAD_LOCAL_TELEMETRY(FastRandomNumberGenerator, TlsRandomNumberGenerator, engine_);
+ } // namespace
+
+-FastRandomNumberGenerator &Random::GetRandomNumberGenerator() noexcept
++FastRandomNumberGenerator *Random::GetRandomNumberGenerator() noexcept
+ {
+- static thread_local TlsRandomNumberGenerator random_number_generator{};
++ // static thread_local TlsRandomNumberGenerator random_number_generator{};
++ BLOCK_STATIC_THREAD_LOCAL_TELEMTRY(TlsRandomNumberGenerator, random_number_generator);
+ return TlsRandomNumberGenerator::engine();
+ }
+
+ uint64_t Random::GenerateRandom64() noexcept
+ {
+- return GetRandomNumberGenerator()();
++ return GetRandomNumberGenerator()->operator()();
+ }
+
+ void Random::GenerateRandomBuffer(opentelemetry::nostd::span<uint8_t> buffer) noexcept
+diff --git a/sdk/src/common/random.h b/sdk/src/common/random.h
+index ecd6dab..1aaa220 100644
+--- a/sdk/src/common/random.h
++++ b/sdk/src/common/random.h
+@@ -34,7 +34,7 @@ class Random
+ /**
+ * @return a seeded thread-local random number generator.
+ */
+- static FastRandomNumberGenerator &GetRandomNumberGenerator() noexcept;
++ static FastRandomNumberGenerator *GetRandomNumberGenerator() noexcept;
+ };
+ } // namespace common
+ } // namespace sdk
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 7485a7ae48..d0121cfc95 100755
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -32,9 +32,6 @@ export TP_SOURCE_DIR=$TP_DIR/src
# thirdparties will be installed to here
export TP_INSTALL_DIR=$TP_DIR/installed
-# libhdfs3-with-kerberos will be installed to here
-export HDFS3_KRB5_INSTALL_DIR=$TP_INSTALL_DIR/libhdfs_with_kerberos
-
# patches for all thirdparties
export TP_PATCH_DIR=$TP_DIR/patches
@@ -48,7 +45,7 @@ export TP_LIB_DIR=$TP_INSTALL_DIR/lib
export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
# source of all dependencies, default unuse it
-# export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
+export REPOSITORY_URL=https://doris-thirdparty-hk-1308700295.cos.ap-hongkong.myqcloud.com/thirdparty
#####################################################
# Download url, filename and unpaced filename
@@ -152,6 +149,18 @@ RE2_NAME=re2-2021-02-02.tar.gz
RE2_SOURCE=re2-2021-02-02
RE2_MD5SUM="48bc665463a86f68243c5af1bac75cd0"
+# hyperscan
+HYPERSCAN_DOWNLOAD="https://github.com/intel/hyperscan/archive/refs/tags/v5.4.0.tar.gz"
+HYPERSCAN_NAME=hyperscan-5.4.0.tar.gz
+HYPERSCAN_SOURCE=hyperscan-5.4.0
+HYPERSCAN_MD5SUM="65e08385038c24470a248f6ff2fa379b"
+
+# ragel (dependency for hyperscan)
+RAGEL_DOWNLOAD="http://www.colm.net/files/ragel/ragel-6.10.tar.gz"
+RAGEL_NAME=ragel-6.10.tar.gz
+RAGEL_SOURCE=ragel-6.10
+RAGEL_MD5SUM="748cae8b50cffe9efcaa5acebc6abf0d"
+
# boost
BOOST_DOWNLOAD="https://boostorg.jfrog.io/artifactory/main/release/1.73.0/source/boost_1_73_0.tar.gz"
BOOST_NAME=boost_1_73_0.tar.gz
@@ -372,12 +381,38 @@ SIMDJSON_NAME=simdjson-1.0.2.tar.gz
SIMDJSON_SOURCE=simdjson-1.0.2
SIMDJSON_MD5SUM="5bb34cca7087a99c450dbdfe406bdc7d"
+# nlohmann_json
+NLOHMANN_JSON_DOWNLOAD="https://github.com/nlohmann/json/archive/refs/tags/v3.10.1.tar.gz"
+NLOHMANN_JSON_NAME=json-3.10.1.tar.gz
+NLOHMANN_JSON_SOURCE=json-3.10.1
+NLOHMANN_JSON_MD5SUM="7b369d567afc0dffdcf5800fd9abb836"
+
+# opentelemetry-proto
+OPENTELEMETRY_PROTO_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-proto/archive/refs/tags/v0.18.0.tar.gz"
+OPENTELEMETRY_PROTO_NAME=opentelemetry-proto-0.18.0.tar.gz
+OPENTELEMETRY_PROTO_SOURCE=opentelemetry-proto-0.18.0
+OPENTELEMETRY_PROTO_MD5SUM="5179f58bb4edbd805590bffd2cf4df85"
+
+# opentelemetry
+OPENTELEMETRY_DOWNLOAD="https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.4.0.tar.gz"
+OPENTELEMETRY_NAME=opentelemetry-cpp-1.4.0.tar.gz
+OPENTELEMETRY_SOURCE=opentelemetry-cpp-1.4.0
+OPENTELEMETRY_MD5SUM="511b670dd1abb596da53684d23742c5f"
+
# libbacktrace
LIBBACKTRACE_DOWNLOAD="https://codeload.github.com/ianlancetaylor/libbacktrace/zip/2446c66076480ce07a6bd868badcbceb3eeecc2e"
LIBBACKTRACE_NAME=libbacktrace-2446c66076480ce07a6bd868badcbceb3eeecc2e.zip
LIBBACKTRACE_SOURCE=libbacktrace-2446c66076480ce07a6bd868badcbceb3eeecc2e
LIBBACKTRACE_MD5SUM="6c79a8012870a24610c0d9c3621b23fe"
+# sse2noen
+SSE2NEON_DOWNLOAD="https://github.com/DLTcollab/sse2neon/archive/refs/tags/v1.5.1.tar.gz"
+SSE2NEON_NAME=sse2neon-1.5.1.tar.gz
+SSE2NEON_SOURCE=sse2neon-1.5.1
+SSE2NEON_MD5SUM="9de5dc2970aa7efac7faee59e2826c51"
+
+
+
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES="LIBEVENT
OPENSSL
@@ -395,6 +430,8 @@ BZIP
LZO2
CURL
RE2
+HYPERSCAN
+RAGEL
BOOST
MYSQL
ODBC
@@ -431,4 +468,8 @@ PDQSORT
BENCHMARK
XSIMD
SIMDJSON
-LIBBACKTRACE"
+NLOHMANN_JSON
+OPENTELEMETRY_PROTO
+OPENTELEMETRY
+LIBBACKTRACE
+SSE2NEON"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org