[doris] branch branch-1.2-lts updated: [deps](libhdfs) add official hadoop libhdfs for x86 (#17435) (#18494)

[mo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
     new 70abac3004 [deps](libhdfs) add official hadoop libhdfs for x86 (#17435) (#18494)
70abac3004 is described below

commit 70abac3004c57d18ed093f56f4b1a13035218b8f
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Sun Apr 9 15:27:15 2023 +0800

    [deps](libhdfs) add official hadoop libhdfs for x86 (#17435) (#18494)
    
    cherry-pick #17435 and #17181
    This is the first step to introduce official hadoop libhdfs to Doris. Because the current hdfs client libhdfs3 lacks some important feature and is hard to maintain.
    
    Download the hadoop 3.3.4 binary from hadoop website: https://hadoop.apache.org/releases.html Extract libs and headers which are used for libhdfs, and pack them into hadoop_lib_3.3.4-x86.tar.gz Upload it to https://github.com/apache/doris-thirdparty/releases/tag/hadoop-libs-3.3.4
    
    TODO:
    The hadoop libs for arm is missing, we need to find a way to build it
---
 thirdparty/build-thirdparty.sh | 17 ++++++++++++++++-
 thirdparty/vars.sh             |  7 +++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 9b9642fff2..1e238e8764 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -856,7 +856,8 @@ build_librdkafka() {
     # PKG_CONFIG="pkg-config --static"
 
     CPPFLAGS="-I${TP_INCLUDE_DIR}" \
-        LDFLAGS="-L${TP_LIB_DIR} -lssl -lcrypto -lzstd -lz -lsasl2" \
+        LDFLAGS="-L${TP_LIB_DIR} -lssl -lcrypto -lzstd -lz -lsasl2 \
+        -lgssapi_krb5 -lkrb5 -lkrb5support -lk5crypto -lcom_err -lresolv" \
         ./configure --prefix="${TP_INSTALL_DIR}" --enable-static --enable-sasl --disable-c11threads
 
     make -j "${PARALLEL}"
@@ -1545,6 +1546,16 @@ build_fast_float() {
     cp -r ./include/fast_float "${TP_INSTALL_DIR}/include/"
 }
 
+# hadoop_libs_x86
+build_hadoop_libs_x86() {
+    check_if_source_exist "${HADOOP_LIBS_X86_SOURCE}"
+    cd "${TP_SOURCE_DIR}/${HADOOP_LIBS_X86_SOURCE}"
+    mkdir -p "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
+    mkdir -p "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
+    cp ./include/hdfs.h "${TP_INSTALL_DIR}/include/hadoop_hdfs/"
+    cp -r ./* "${TP_INSTALL_DIR}/lib/hadoop_hdfs/"
+}
+
 if [[ "$(uname -s)" == 'Darwin' ]]; then
     echo 'build for Darwin'
     build_binutils
@@ -1608,4 +1619,8 @@ build_xxhash
 build_concurrentqueue
 build_fast_float
 
+if [[ "$(uname -m)" == 'x86_64' ]]; then
+    build_hadoop_libs_x86
+fi
+
 echo "Finished to build all thirdparties"
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index a41eefd899..2535b8eea6 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -441,6 +441,12 @@ FAST_FLOAT_NAME=fast_float-3.9.0.tar.gz
 FAST_FLOAT_SOURCE=fast_float-3.9.0
 FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea"
 
+# libhdfs
+HADOOP_LIBS_X86_DOWNLOAD="https://github.com/apache/doris-thirdparty/releases/download/hadoop-libs-3.3.4/hadoop_lib_3.3.4-x86.tar.gz"
+HADOOP_LIBS_X86_NAME="hadoop_lib_3.3.4-x86.tar.gz"
+HADOOP_LIBS_X86_SOURCE="hadoop_lib_3.3.4-x86"
+HADOOP_LIBS_X86_MD5SUM="96117450170487f007ffeca5ddf62f7e"
+
 # all thirdparties which need to be downloaded is set in array TP_ARCHIVES
 export TP_ARCHIVES=(
     'LIBEVENT'
@@ -505,6 +511,7 @@ export TP_ARCHIVES=(
     'XXHASH'
     'CONCURRENTQUEUE'
     'FAST_FLOAT'
+    'HADOOP_LIBS_X86'
 )
 
 if [[ "$(uname -s)" == 'Darwin' ]]; then


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org