You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/22 12:58:24 UTC
[incubator-doris] branch master updated: [deps] libhdfs3 build enable kerberos support (#9524)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new b3a2a92bf5 [deps] libhdfs3 build enable kerberos support (#9524)
b3a2a92bf5 is described below
commit b3a2a92bf550fc039b27f045bd058549903071a6
Author: gtchaos <gs...@gmail.com>
AuthorDate: Sun May 22 20:58:19 2022 +0800
[deps] libhdfs3 build enable kerberos support (#9524)
Currently, the libhdfs3 library integrated by doris BE does not support accessing the cluster with kerberos authentication
enabled, and found that kerberos-related dependencies（gsasl and krb5） were not added when build libhdfs3.
so, this pr will enable kerberos support and rebuild libhdfs3 with dependencies gsasl and krb5:
- gsasl version: 1.8.0
- krb5 version: 1.19
---
be/CMakeLists.txt | 50 +++++++++++++++++++++++++++++----
build.sh | 6 +++-
run-be-ut.sh | 1 +
thirdparty/build-thirdparty.sh | 37 ++++++++++++++++++++++--
thirdparty/download-thirdparty.sh | 12 +++++++-
thirdparty/patches/libgsasl-1.8.0.patch | 24 ++++++++++++++++
thirdparty/vars.sh | 20 +++++++++++--
7 files changed, 139 insertions(+), 11 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index bae0965f3f..76edc3ba27 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -85,6 +85,7 @@ set(BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
set(ENV{DORIS_HOME} "${BASE_DIR}/../")
set(BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed/")
+set(HDFS3_KRB5_INSTALL_DIR "$ENV{DORIS_THIRDPARTY}/installed/libhdfs_with_kerberos/")
set(GENSRC_DIR "${BASE_DIR}/../gensrc/build/")
set(SRC_DIR "${BASE_DIR}/src/")
set(TEST_DIR "${CMAKE_SOURCE_DIR}/test/")
@@ -97,6 +98,7 @@ else()
endif()
message(STATUS "make test: ${MAKE_TEST}")
option(WITH_MYSQL "Support access MySQL" ON)
+option(WITH_KERBEROS "Enable or disable Kereberos support" ${WITH_KERBEROS})
# Check gcc
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -328,17 +330,40 @@ set_target_properties(minizip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
add_library(idn STATIC IMPORTED)
set_target_properties(idn PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libidn.a)
-add_library(gsasl STATIC IMPORTED)
-set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
-
add_library(breakpad STATIC IMPORTED)
set_target_properties(breakpad PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libbreakpad_client.a)
-if (ARCH_AMD64)
- # libhdfs3 only support x86 or amd64
+if (WITH_KERBEROS)
+ # kerberos lib for libhdfs3
+ add_library(gsasl STATIC IMPORTED)
+ set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgsasl.a)
+
+ add_library(krb5support STATIC IMPORTED)
+ set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5support.a)
+
+ add_library(krb5 STATIC IMPORTED)
+ set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5.a)
+
+ add_library(com_err STATIC IMPORTED)
+ set_target_properties(com_err PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libcom_err.a)
+
+ add_library(k5crypto STATIC IMPORTED)
+ set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libk5crypto.a)
+
+ add_library(gssapi_krb5 STATIC IMPORTED)
+ set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgssapi_krb5.a)
+
+ add_library(hdfs3 STATIC IMPORTED)
+ set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libhdfs3.a)
+else()
+ add_library(gsasl STATIC IMPORTED)
+ set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
+
add_library(hdfs3 STATIC IMPORTED)
set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a)
+endif()
+if (ARCH_AMD64)
add_library(xml2 STATIC IMPORTED)
set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)
@@ -507,6 +532,14 @@ endif()
set(WL_START_GROUP "-Wl,--start-group")
set(WL_END_GROUP "-Wl,--end-group")
+
+set(KRB5_LIBS
+ krb5support
+ krb5
+ com_err
+ gssapi_krb5
+ k5crypto)
+
set(AWS_LIBS
aws-sdk-s3
aws-sdk-core
@@ -640,6 +673,12 @@ if (WITH_MYSQL)
)
endif()
+if (WITH_KERBEROS)
+ set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
+ ${KRB5_LIBS}
+ )
+endif()
+
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
message(STATUS "DORIS_DEPENDENCIES is ${DORIS_DEPENDENCIES}")
@@ -651,6 +690,7 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
-static-libstdc++
-static-libgcc
-lstdc++fs
+ -lresolv
)
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
diff --git a/build.sh b/build.sh
index c5f91dda40..e3b3243eb8 100755
--- a/build.sh
+++ b/build.sh
@@ -175,7 +175,9 @@ if [[ ${HELP} -eq 1 ]]; then
usage
exit
fi
-
+if [[ -z ${WITH_KERBEROS} ]]; then
+ WITH_KERBEROS=ON
+fi
# build thirdparty libraries if necessary
if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a ]]; then
echo "Thirdparty libraries need to be build ..."
@@ -228,6 +230,7 @@ echo "Get params:
PARALLEL -- $PARALLEL
CLEAN -- $CLEAN
WITH_MYSQL -- $WITH_MYSQL
+ WITH_KERBEROS -- $WITH_KERBEROS
WITH_LZO -- $WITH_LZO
GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY
USE_AVX2 -- $USE_AVX2
@@ -289,6 +292,7 @@ if [ ${BUILD_BE} -eq 1 ] ; then
-DMAKE_TEST=OFF \
${CMAKE_USE_CCACHE} \
-DWITH_MYSQL=${WITH_MYSQL} \
+ -DWITH_KERBEROS=${WITH_KERBEROS} \
-DWITH_LZO=${WITH_LZO} \
-DUSE_LIBCPP=${USE_LIBCPP} \
-DBUILD_META_TOOL=${BUILD_META_TOOL} \
diff --git a/run-be-ut.sh b/run-be-ut.sh
index 76854e2ebe..a8b0950cce 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -136,6 +136,7 @@ ${CMAKE_CMD} -G "${GENERATOR}" \
-DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
-DBUILD_META_TOOL=OFF \
-DWITH_MYSQL=OFF \
+ -DWITH_KERBEROS=OFF \
-DUSE_DWARF=${USE_DWARF} \
${CMAKE_USE_CCACHE} ../
${BUILD_SYSTEM} -j ${PARALLEL}
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 3b7a108c28..d29ae4bf8e 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -904,16 +904,46 @@ build_gsasl() {
make -j $PARALLEL && make install
}
+# build_gsasl2 just for libgsasl1.8.0
+build_gsasl2() {
+ check_if_source_exist $GSASL2_SOURCE
+ cd $TP_SOURCE_DIR/$GSASL2_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --with-gssapi-impl=mit --enable-shared=no --with-pic --with-libidn-prefix=$TP_INSTALL_DIR
+ make -j $PARALLEL && make install
+}
+
+# krb5
+build_krb5() {
+ check_if_source_exist $KRB5_SOURCE
+ cd $TP_SOURCE_DIR/$KRB5_SOURCE/src
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ CFLAGS="-fcommon" \
+ ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --disable-shared --enable-static
+ make -j $PARALLEL && make install
+}
+
# hdfs3
build_hdfs3() {
check_if_source_exist $HDFS3_SOURCE
cd $TP_SOURCE_DIR/$HDFS3_SOURCE
- mkdir -p $BUILD_DIR && cd $BUILD_DIR
- ../bootstrap --dependency=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
+ # build libhdfs3 without kerberos
+ ../bootstrap --dependency="$TP_INSTALL_DIR" --prefix=$TP_INSTALL_DIR
make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
make install
}
+# hdfs3_with_kerberos
+build_hdfs3_with_kerberos() {
+ check_if_source_exist $HDFS3_SOURCE
+ cd $TP_SOURCE_DIR/$HDFS3_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
+ # build libhdfs3 with kerberos support
+ ../bootstrap --dependency="$HDFS3_KRB5_INSTALL_DIR:$TP_INSTALL_DIR -DWITH_KERBEROS=true" --prefix=$HDFS3_KRB5_INSTALL_DIR
+ make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
+ make install
+}
# benchmark
build_benchmark() {
check_if_source_exist $BENCHMARK_SOURCE
@@ -999,7 +1029,10 @@ build_lzma
build_xml2
build_idn
build_gsasl
+build_gsasl2
+build_krb5
build_hdfs3
+build_hdfs3_with_kerberos
build_benchmark
build_breakpad
build_simdjson
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index b17c6be0bd..6014611ab5 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -156,7 +156,7 @@ do
done
echo "===== Downloading thirdparty archives...done"
-# check if all tp archievs exists
+# check if all tp archives exists
echo "===== Checking all thirdpart archives..."
for TP_ARCH in ${TP_ARCHIVES[*]}
do
@@ -271,6 +271,16 @@ fi
cd -
echo "Finished patching $S2_SOURCE"
+# gsasl2 patch to fix link error such as mutilple func defination
+# when link target with kerberos
+cd $TP_SOURCE_DIR/$GSASL2_SOURCE
+if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/libgsasl-1.8.0.patch
+ touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $GSASL2_SOURCE"
+
# hdfs3 patch to fix compile error
cd $TP_SOURCE_DIR/$HDFS3_SOURCE
if [ ! -f $PATCHED_MARK ]; then
diff --git a/thirdparty/patches/libgsasl-1.8.0.patch b/thirdparty/patches/libgsasl-1.8.0.patch
new file mode 100644
index 0000000000..7fe5ebaeee
--- /dev/null
+++ b/thirdparty/patches/libgsasl-1.8.0.patch
@@ -0,0 +1,24 @@
+--- a/config.h.in 2012-05-29 01:11:28.000000000 +0800
++++ b/config.h.in 2022-05-11 18:06:02.000000000 +0800
+@@ -176,7 +176,7 @@
+ #undef HAVE_GSSAPI_H
+
+ /* Define to 1 if you have the `GSS_C_NT_HOSTBASED_SERVICE' function. */
+-#undef HAVE_GSS_C_NT_HOSTBASED_SERVICE
++#define HAVE_GSS_C_NT_HOSTBASED_SERVICE 1
+
+ /* Define to 1 if you have the `gss_decapsulate_token' function. */
+ #undef HAVE_GSS_DECAPSULATE_TOKEN
+@@ -185,10 +185,10 @@
+ #undef HAVE_GSS_ENCAPSULATE_TOKEN
+
+ /* Define to 1 if you have the `gss_inquire_mech_for_saslname' function. */
+-#undef HAVE_GSS_INQUIRE_MECH_FOR_SASLNAME
++#define HAVE_GSS_INQUIRE_MECH_FOR_SASLNAME 1
+
+ /* Define to 1 if you have the `gss_oid_equal' function. */
+-#undef HAVE_GSS_OID_EQUAL
++#define HAVE_GSS_OID_EQUAL 1
+
+ /* Define if you have the iconv() function and it works. */
+ #undef HAVE_ICONV
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 65dbc0df23..12ece01216 100755
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -32,6 +32,9 @@ export TP_SOURCE_DIR=$TP_DIR/src
# thirdparties will be installed to here
export TP_INSTALL_DIR=$TP_DIR/installed
+# libhdfs3-with-kerberos will be installed to here
+export HDFS3_KRB5_INSTALL_DIR=$TP_INSTALL_DIR/libhdfs_with_kerberos
+
# patches for all thirdparties
export TP_PATCH_DIR=$TP_DIR/patches
@@ -44,8 +47,8 @@ export TP_LIB_DIR=$TP_INSTALL_DIR/lib
# all java libraries will be unpacked to here
export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
-# source of all dependencies
-export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
+# source of all dependencies, default unuse it
+# export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
#####################################################
# Download url, filename and unpaced filename
@@ -326,6 +329,17 @@ GSASL_NAME="libgsasl-1.10.0.tar.gz"
GSASL_SOURCE="libgsasl-1.10.0"
GSASL_MD5SUM="9c8fc632da4ce108fb7581b33de2a5ce"
+GSASL2_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz"
+GSASL2_NAME="libgsasl-1.8.0.tar.gz"
+GSASL2_SOURCE="libgsasl-1.8.0"
+GSASL2_MD5SUM="5dbdf859f6e60e05813370e2b193b92b"
+
+# krb5
+KRB5_DOWNLOAD="https://kerberos.org/dist/krb5/1.19/krb5-1.19.tar.gz"
+KRB5_NAME="krb5-1.19.tar.gz"
+KRB5_SOURCE="krb5-1.19"
+KRB5_MD5SUM="aaf18447a5a014aa3b7e81814923f4c9"
+
# hdfs3
HDFS3_DOWNLOAD="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/libhdfs3-master.zip"
HDFS3_NAME="libhdfs3-master.zip"
@@ -423,6 +437,8 @@ LZMA
XML2
IDN
GSASL
+GSASL2
+KRB5
HDFS3
LIBDIVIDE
PDQSORT
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org