You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/10 11:40:47 UTC
[incubator-doris] branch master updated: [dependency][enhancement] support build libhdfs in arm cpus (#10018)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e0cf2677a0 [dependency][enhancement] support build libhdfs in arm cpus (#10018)
e0cf2677a0 is described below
commit e0cf2677a02db31f386ccf72a760b22e825af315
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Fri Jun 10 19:40:41 2022 +0800
[dependency][enhancement] support build libhdfs in arm cpus (#10018)
Supports native hdfs functionality on arm cpu
This pr mainly upgrades libdfs3 and supports running on arm，and make libhdfs3 with kerberos as default
---
be/CMakeLists.txt | 84 ++++++-------------
be/src/exec/CMakeLists.txt | 18 ----
be/src/io/CMakeLists.txt | 22 ++---
be/src/io/broker_reader.cpp | 2 +-
be/src/io/broker_reader.h | 2 +-
be/src/io/broker_writer.cpp | 2 +-
be/src/io/broker_writer.h | 2 +-
be/src/io/buffered_reader.cpp | 2 +-
be/src/io/buffered_reader.h | 2 +-
be/src/io/file_factory.cpp | 18 ++--
be/src/io/file_factory.h | 4 +-
be/src/io/hdfs_file_reader.cpp | 2 +-
be/src/io/hdfs_file_reader.h | 2 +-
be/src/io/hdfs_reader_writer.cpp | 16 +---
be/src/io/hdfs_reader_writer.h | 4 +-
be/src/io/hdfs_writer.cpp | 2 +-
be/src/io/hdfs_writer.h | 2 +-
be/src/io/local_file_reader.cpp | 2 +-
be/src/io/local_file_reader.h | 2 +-
be/src/io/local_file_writer.cpp | 2 +-
be/src/io/local_file_writer.h | 2 +-
be/src/io/s3_reader.cpp | 2 +-
be/src/io/s3_reader.h | 2 +-
be/src/io/s3_writer.cpp | 2 +-
be/src/io/s3_writer.h | 2 +-
build.sh | 5 --
dist/LICENSE-dist.txt | 1 +
run-be-ut.sh | 1 -
thirdparty/CHANGELOG.md | 5 ++
thirdparty/build-thirdparty.sh | 33 ++------
thirdparty/download-thirdparty.sh | 13 +--
thirdparty/patches/libhdfs3-master.patch | 138 -------------------------------
thirdparty/vars.sh | 26 ++----
33 files changed, 92 insertions(+), 332 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 480a3c6e46..999801ea26 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -85,7 +85,6 @@ set(BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
set(ENV{DORIS_HOME} "${BASE_DIR}/../")
set(BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed/")
-set(HDFS3_KRB5_INSTALL_DIR "$ENV{DORIS_THIRDPARTY}/installed/libhdfs_with_kerberos/")
set(GENSRC_DIR "${BASE_DIR}/../gensrc/build/")
set(SRC_DIR "${BASE_DIR}/src/")
set(TEST_DIR "${CMAKE_SOURCE_DIR}/test/")
@@ -98,7 +97,6 @@ else()
endif()
message(STATUS "make test: ${MAKE_TEST}")
option(WITH_MYSQL "Support access MySQL" ON)
-option(WITH_KERBEROS "Enable or disable Kereberos support" ${WITH_KERBEROS})
# Check gcc
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -330,43 +328,32 @@ set_target_properties(minizip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
add_library(idn STATIC IMPORTED)
set_target_properties(idn PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libidn.a)
-if (WITH_KERBEROS)
- # kerberos lib for libhdfs3
- add_library(gsasl STATIC IMPORTED)
- set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgsasl.a)
+add_library(xml2 STATIC IMPORTED)
+set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)
- add_library(krb5support STATIC IMPORTED)
- set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5support.a)
-
- add_library(krb5 STATIC IMPORTED)
- set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5.a)
+add_library(lzma STATIC IMPORTED)
+set_target_properties(lzma PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/liblzma.a)
- add_library(com_err STATIC IMPORTED)
- set_target_properties(com_err PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libcom_err.a)
+add_library(gsasl STATIC IMPORTED)
+set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libgsasl.a)
- add_library(k5crypto STATIC IMPORTED)
- set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libk5crypto.a)
+add_library(krb5support STATIC IMPORTED)
+set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libkrb5support.a)
- add_library(gssapi_krb5 STATIC IMPORTED)
- set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgssapi_krb5.a)
+add_library(krb5 STATIC IMPORTED)
+set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libkrb5.a)
- add_library(hdfs3 STATIC IMPORTED)
- set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libhdfs3.a)
-else()
- add_library(gsasl STATIC IMPORTED)
- set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
+add_library(com_err STATIC IMPORTED)
+set_target_properties(com_err PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libcom_err.a)
- add_library(hdfs3 STATIC IMPORTED)
- set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a)
-endif()
+add_library(k5crypto STATIC IMPORTED)
+set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libk5crypto.a)
-if (ARCH_AMD64)
- add_library(xml2 STATIC IMPORTED)
- set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)
+add_library(gssapi_krb5 STATIC IMPORTED)
+set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libgssapi_krb5.a)
- add_library(lzma STATIC IMPORTED)
- set_target_properties(lzma PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/liblzma.a)
-endif()
+add_library(hdfs3 STATIC IMPORTED)
+set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libhdfs3.a)
find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)
@@ -570,6 +557,7 @@ set(DORIS_LINK_LIBS
Exec
Exprs
Gutil
+ IO
Olap
Rowset
OlapFs
@@ -632,6 +620,9 @@ set(COMMON_THIRDPARTY
# put this after lz4 to avoid using lz4 lib in librdkafka
librdkafka_cpp
librdkafka
+ hdfs3
+ xml2
+ lzma
)
if (${MAKE_TEST} STREQUAL "ON")
@@ -641,29 +632,13 @@ if (${MAKE_TEST} STREQUAL "ON")
)
endif()
-# thirdparties dependescies that can only run on X86 platform
-set(X86_DEPENDENCIES
+set(DORIS_DEPENDENCIES
+ ${DORIS_DEPENDENCIES}
+ ${WL_START_GROUP}
${COMMON_THIRDPARTY}
- hdfs3
- xml2
- lzma
+ ${KRB5_LIBS}
)
-if(ARCH_AARCH64)
- # Set thirdparty libraries
- set(DORIS_DEPENDENCIES
- ${DORIS_DEPENDENCIES}
- ${WL_START_GROUP}
- ${COMMON_THIRDPARTY}
- )
-else()
- set(DORIS_DEPENDENCIES
- ${DORIS_DEPENDENCIES}
- ${WL_START_GROUP}
- ${X86_DEPENDENCIES}
- )
-endif()
-
if(WITH_LZO)
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
lzo
@@ -676,12 +651,6 @@ if (WITH_MYSQL)
)
endif()
-if (WITH_KERBEROS)
- set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
- ${KRB5_LIBS}
- )
-endif()
-
set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
message(STATUS "DORIS_DEPENDENCIES is ${DORIS_DEPENDENCIES}")
@@ -755,6 +724,7 @@ add_subdirectory(${SRC_DIR}/exec)
add_subdirectory(${SRC_DIR}/exprs)
add_subdirectory(${SRC_DIR}/gen_cpp)
add_subdirectory(${SRC_DIR}/geo)
+add_subdirectory(${SRC_DIR}/io)
add_subdirectory(${SRC_DIR}/gutil)
add_subdirectory(${SRC_DIR}/http)
add_subdirectory(${SRC_DIR}/olap)
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index 5708eb0465..cc94f5fe9a 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -28,7 +28,6 @@ set(EXEC_FILES
analytic_eval_node.cpp
blocking_join_node.cpp
broker_scan_node.cpp
- ../io/buffered_reader.cpp
base_scanner.cpp
broker_scanner.cpp
cross_join_node.cpp
@@ -98,24 +97,7 @@ set(EXEC_FILES
json_scanner.cpp
assert_num_rows_node.cpp
- ../io/local_file_reader.cpp
- ../io/local_file_writer.cpp
- ../io/broker_reader.cpp
- ../io/broker_writer.cpp
- ../io/s3_reader.cpp
- ../io/s3_writer.cpp
- ../io/hdfs_reader_writer.cpp
- ../io/file_factory.cpp
)
-
-if (ARCH_AMD64)
- set(EXEC_FILES
- ${EXEC_FILES}
- ../io/hdfs_file_reader.cpp
- ../io/hdfs_writer.cpp
- )
-endif()
-
if (WITH_MYSQL)
set(EXEC_FILES
${EXEC_FILES}
diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt
index 1be0e05176..286a9471db 100644
--- a/be/src/io/CMakeLists.txt
+++ b/be/src/io/CMakeLists.txt
@@ -21,26 +21,20 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io")
# where to put generated binaries
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io")
-set(EXEC_FILES
+set(IO_FILES
+ broker_reader.cpp
+ broker_writer.cpp
buffered_reader.cpp
+ file_factory.cpp
+ hdfs_file_reader.cpp
+ hdfs_reader_writer.cpp
+ hdfs_writer.cpp
local_file_reader.cpp
local_file_writer.cpp
- broker_reader.cpp
- broker_writer.cpp
s3_reader.cpp
s3_writer.cpp
- hdfs_reader_writer.cpp
- file_factory.cpp
)
-if (ARCH_AMD64)
- set(EXEC_FILES
- ${EXEC_FILES}
- hdfs_file_reader.cpp
- hdfs_writer.cpp
- )
-endif()
-
add_library(IO STATIC
- ${EXEC_FILES}
+ ${IO_FILES}
)
diff --git a/be/src/io/broker_reader.cpp b/be/src/io/broker_reader.cpp
index 1745a7bbc2..173a1c8de4 100644
--- a/be/src/io/broker_reader.cpp
+++ b/be/src/io/broker_reader.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "broker_reader.h"
+#include "io/broker_reader.h"
#include <sstream>
diff --git a/be/src/io/broker_reader.h b/be/src/io/broker_reader.h
index 94f86ca8df..1c48fc4381 100644
--- a/be/src/io/broker_reader.h
+++ b/be/src/io/broker_reader.h
@@ -23,9 +23,9 @@
#include <string>
#include "common/status.h"
-#include "file_reader.h"
#include "gen_cpp/PaloBrokerService_types.h"
#include "gen_cpp/Types_types.h"
+#include "io/file_reader.h"
namespace doris {
diff --git a/be/src/io/broker_writer.cpp b/be/src/io/broker_writer.cpp
index 3975abe664..419e962b68 100644
--- a/be/src/io/broker_writer.cpp
+++ b/be/src/io/broker_writer.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "broker_writer.h"
+#include "io/broker_writer.h"
#include <sstream>
diff --git a/be/src/io/broker_writer.h b/be/src/io/broker_writer.h
index c4b8bac17c..1155fd4e9e 100644
--- a/be/src/io/broker_writer.h
+++ b/be/src/io/broker_writer.h
@@ -23,9 +23,9 @@
#include <string>
#include "common/status.h"
-#include "file_writer.h"
#include "gen_cpp/PaloBrokerService_types.h"
#include "gen_cpp/Types_types.h"
+#include "io/file_writer.h"
namespace doris {
diff --git a/be/src/io/buffered_reader.cpp b/be/src/io/buffered_reader.cpp
index 5a80c4f842..5125facddc 100644
--- a/be/src/io/buffered_reader.cpp
+++ b/be/src/io/buffered_reader.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "buffered_reader.h"
+#include "io/buffered_reader.h"
#include <algorithm>
#include <sstream>
diff --git a/be/src/io/buffered_reader.h b/be/src/io/buffered_reader.h
index 8ffd5cd0ab..c853606bd5 100644
--- a/be/src/io/buffered_reader.h
+++ b/be/src/io/buffered_reader.h
@@ -22,7 +22,7 @@
#include <memory>
#include "common/status.h"
-#include "file_reader.h"
+#include "io/file_reader.h"
#include "olap/olap_define.h"
#include "util/runtime_profile.h"
diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp
index f8c86a9146..9937eaa6d5 100644
--- a/be/src/io/file_factory.cpp
+++ b/be/src/io/file_factory.cpp
@@ -15,18 +15,18 @@
// specific language governing permissions and limitations
// under the License.
-#include "file_factory.h"
+#include "io/file_factory.h"
-#include "broker_reader.h"
-#include "broker_writer.h"
-#include "buffered_reader.h"
-#include "hdfs_reader_writer.h"
-#include "local_file_reader.h"
-#include "local_file_writer.h"
+#include "io/broker_reader.h"
+#include "io/broker_writer.h"
+#include "io/buffered_reader.h"
+#include "io/hdfs_reader_writer.h"
+#include "io/local_file_reader.h"
+#include "io/local_file_writer.h"
+#include "io/s3_reader.h"
+#include "io/s3_writer.h"
#include "runtime/exec_env.h"
#include "runtime/stream_load/load_stream_mgr.h"
-#include "s3_reader.h"
-#include "s3_writer.h"
doris::Status doris::FileFactory::create_file_writer(
TFileType::type type, doris::ExecEnv* env,
diff --git a/be/src/io/file_factory.h b/be/src/io/file_factory.h
index 36c2871599..c1350d6682 100644
--- a/be/src/io/file_factory.h
+++ b/be/src/io/file_factory.h
@@ -16,10 +16,10 @@
// under the License.
#pragma once
-#include "file_reader.h"
-#include "file_writer.h"
#include "gen_cpp/PlanNodes_types.h"
#include "gen_cpp/Types_types.h"
+#include "io/file_reader.h"
+#include "io/file_writer.h"
namespace doris {
class ExecEnv;
diff --git a/be/src/io/hdfs_file_reader.cpp b/be/src/io/hdfs_file_reader.cpp
index 5d4dc2cc5e..bce0b6b0fe 100644
--- a/be/src/io/hdfs_file_reader.cpp
+++ b/be/src/io/hdfs_file_reader.cpp
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#include "hdfs_file_reader.h"
+#include "io/hdfs_file_reader.h"
#include <sys/stat.h>
#include <unistd.h>
diff --git a/be/src/io/hdfs_file_reader.h b/be/src/io/hdfs_file_reader.h
index 83c8efb7dc..67fa60a7d3 100644
--- a/be/src/io/hdfs_file_reader.h
+++ b/be/src/io/hdfs_file_reader.h
@@ -19,8 +19,8 @@
#include <hdfs/hdfs.h>
-#include "file_reader.h"
#include "gen_cpp/PlanNodes_types.h"
+#include "io/file_reader.h"
namespace doris {
diff --git a/be/src/io/hdfs_reader_writer.cpp b/be/src/io/hdfs_reader_writer.cpp
index 9a072512c0..a93ba02747 100644
--- a/be/src/io/hdfs_reader_writer.cpp
+++ b/be/src/io/hdfs_reader_writer.cpp
@@ -15,34 +15,24 @@
// specific language governing permissions and limitations
// under the License.
-#include "hdfs_reader_writer.h"
+#include "io/hdfs_reader_writer.h"
-#if defined(__x86_64__)
-#include "hdfs_file_reader.h"
-#include "hdfs_writer.h"
-#endif
+#include "io/hdfs_file_reader.h"
+#include "io/hdfs_writer.h"
namespace doris {
Status HdfsReaderWriter::create_reader(const THdfsParams& hdfs_params, const std::string& path,
int64_t start_offset, FileReader** reader) {
-#if defined(__x86_64__)
*reader = new HdfsFileReader(hdfs_params, path, start_offset);
return Status::OK();
-#else
- return Status::InternalError("HdfsFileReader do not support on non x86 platform");
-#endif
}
Status HdfsReaderWriter::create_writer(std::map<std::string, std::string>& properties,
const std::string& path,
std::unique_ptr<FileWriter>& writer) {
-#if defined(__x86_64__)
writer.reset(new HDFSWriter(properties, path));
return Status::OK();
-#else
- return Status::InternalError("HdfsWriter do not support on non x86 platform");
-#endif
}
} // namespace doris
diff --git a/be/src/io/hdfs_reader_writer.h b/be/src/io/hdfs_reader_writer.h
index 160306ffce..9177eaf240 100644
--- a/be/src/io/hdfs_reader_writer.h
+++ b/be/src/io/hdfs_reader_writer.h
@@ -17,9 +17,9 @@
#pragma once
-#include "file_reader.h"
-#include "file_writer.h"
#include "gen_cpp/PlanNodes_types.h"
+#include "io/file_reader.h"
+#include "io/file_writer.h"
namespace doris {
diff --git a/be/src/io/hdfs_writer.cpp b/be/src/io/hdfs_writer.cpp
index 16b2516ca8..6727d9a6d9 100644
--- a/be/src/io/hdfs_writer.cpp
+++ b/be/src/io/hdfs_writer.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "hdfs_writer.h"
+#include "io/hdfs_writer.h"
#include <filesystem>
diff --git a/be/src/io/hdfs_writer.h b/be/src/io/hdfs_writer.h
index 15d9a1fde8..e26a3eb7ad 100644
--- a/be/src/io/hdfs_writer.h
+++ b/be/src/io/hdfs_writer.h
@@ -22,7 +22,7 @@
#include <map>
#include <string>
-#include "file_writer.h"
+#include "io/file_writer.h"
namespace doris {
class HDFSWriter : public FileWriter {
diff --git a/be/src/io/local_file_reader.cpp b/be/src/io/local_file_reader.cpp
index e8e2d38ea2..514d3483cf 100644
--- a/be/src/io/local_file_reader.cpp
+++ b/be/src/io/local_file_reader.cpp
@@ -14,7 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#include "local_file_reader.h"
+#include "io/local_file_reader.h"
#include <sys/stat.h>
#include <unistd.h>
diff --git a/be/src/io/local_file_reader.h b/be/src/io/local_file_reader.h
index c525804395..0f712814db 100644
--- a/be/src/io/local_file_reader.h
+++ b/be/src/io/local_file_reader.h
@@ -20,7 +20,7 @@
#define _FILE_OFFSET_BITS 64
#include <stdio.h>
-#include "file_reader.h"
+#include "io/file_reader.h"
namespace doris {
diff --git a/be/src/io/local_file_writer.cpp b/be/src/io/local_file_writer.cpp
index 3c425a46aa..e73dc96ebe 100644
--- a/be/src/io/local_file_writer.cpp
+++ b/be/src/io/local_file_writer.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "local_file_writer.h"
+#include "io/local_file_writer.h"
#include "service/backend_options.h"
#include "util/error_util.h"
diff --git a/be/src/io/local_file_writer.h b/be/src/io/local_file_writer.h
index 7d9da485c3..48ba05764b 100644
--- a/be/src/io/local_file_writer.h
+++ b/be/src/io/local_file_writer.h
@@ -19,7 +19,7 @@
#include <stdio.h>
-#include "file_writer.h"
+#include "io/file_writer.h"
namespace doris {
diff --git a/be/src/io/s3_reader.cpp b/be/src/io/s3_reader.cpp
index c932e2d886..a1c6b8cd2c 100644
--- a/be/src/io/s3_reader.cpp
+++ b/be/src/io/s3_reader.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "s3_reader.h"
+#include "io/s3_reader.h"
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
diff --git a/be/src/io/s3_reader.h b/be/src/io/s3_reader.h
index a1464324df..e1fcf2675e 100644
--- a/be/src/io/s3_reader.h
+++ b/be/src/io/s3_reader.h
@@ -20,7 +20,7 @@
#include <map>
#include <string>
-#include "file_reader.h"
+#include "io/file_reader.h"
#include "util/s3_uri.h"
namespace Aws {
diff --git a/be/src/io/s3_writer.cpp b/be/src/io/s3_writer.cpp
index df37e7d820..8c0ddd6696 100644
--- a/be/src/io/s3_writer.cpp
+++ b/be/src/io/s3_writer.cpp
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-#include "s3_writer.h"
+#include "io/s3_writer.h"
#include <aws/core/utils/FileSystemUtils.h>
#include <aws/s3/S3Client.h>
diff --git a/be/src/io/s3_writer.h b/be/src/io/s3_writer.h
index ae2756da08..a0ad9954a9 100644
--- a/be/src/io/s3_writer.h
+++ b/be/src/io/s3_writer.h
@@ -20,7 +20,7 @@
#include <map>
#include <string>
-#include "file_writer.h"
+#include "io/file_writer.h"
#include "util/s3_uri.h"
namespace Aws {
diff --git a/build.sh b/build.sh
index 907f5510ff..8ece656d7a 100755
--- a/build.sh
+++ b/build.sh
@@ -175,9 +175,6 @@ if [[ ${HELP} -eq 1 ]]; then
usage
exit
fi
-if [[ -z ${WITH_KERBEROS} ]]; then
- WITH_KERBEROS=ON
-fi
# build thirdparty libraries if necessary
if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a ]]; then
echo "Thirdparty libraries need to be build ..."
@@ -233,7 +230,6 @@ echo "Get params:
PARALLEL -- $PARALLEL
CLEAN -- $CLEAN
WITH_MYSQL -- $WITH_MYSQL
- WITH_KERBEROS -- $WITH_KERBEROS
WITH_LZO -- $WITH_LZO
GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY
USE_AVX2 -- $USE_AVX2
@@ -296,7 +292,6 @@ if [ ${BUILD_BE} -eq 1 ] ; then
-DMAKE_TEST=OFF \
${CMAKE_USE_CCACHE} \
-DWITH_MYSQL=${WITH_MYSQL} \
- -DWITH_KERBEROS=${WITH_KERBEROS} \
-DWITH_LZO=${WITH_LZO} \
-DUSE_LIBCPP=${USE_LIBCPP} \
-DBUILD_META_TOOL=${BUILD_META_TOOL} \
diff --git a/dist/LICENSE-dist.txt b/dist/LICENSE-dist.txt
index 2d9bfe9016..bb3840be85 100644
--- a/dist/LICENSE-dist.txt
+++ b/dist/LICENSE-dist.txt
@@ -1524,6 +1524,7 @@ The Apache Software License, Version 2.0
* aws sdk: 1.9.211
* benchmark: 1.5.6
* simdjson: 1.0.2
+ * libhdfs3: 2.3.0
* libhdfs3: commit 5fccd36
* opentelemetry-proto: 0.18.0
* opentelemetry-cpp: 1.4.0
diff --git a/run-be-ut.sh b/run-be-ut.sh
index d075693454..19d52865f4 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -136,7 +136,6 @@ ${CMAKE_CMD} -G "${GENERATOR}" \
-DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
-DBUILD_META_TOOL=OFF \
-DWITH_MYSQL=OFF \
- -DWITH_KERBEROS=OFF \
-DUSE_DWARF=${USE_DWARF} \
-DUSE_MEM_TRACKER=ON \
${CMAKE_USE_CCACHE} ../
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
index 578eb7ffc0..c94b73a982 100644
--- a/thirdparty/CHANGELOG.md
+++ b/thirdparty/CHANGELOG.md
@@ -2,6 +2,11 @@
This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/incubator-doris, and the tag is `build-env-${version}`
+## v20220608
+- Remove: remove libhdfs3 without kerberos support
+- Modified: make libhdfs3 with kerberos support as default
+- Modified: change libhdfs3 to https://github.com/yangzhg/libhdfs3/releases/tag/v2.3.0 . This version support arm CPUs
+
## v20220607
- Added: opentelemetry-cpp 1.4.0, it was introduced for tracing.
- Added: opentelemetry-proto 0.18.0, it is depended on by opentelemetry-cpp.
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index dfd3f0012b..6f5dc53590 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -680,6 +680,7 @@ build_arrow() {
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DARROW_BOOST_USE_SHARED=OFF \
+ -DBoost_USE_STATIC_RUNTIME=ON \
-DARROW_GFLAGS_USE_SHARED=OFF \
-Dgflags_ROOT=$TP_INSTALL_DIR \
-DGLOG_ROOT=$TP_INSTALL_DIR \
@@ -938,16 +939,7 @@ build_gsasl() {
check_if_source_exist $GSASL_SOURCE
cd $TP_SOURCE_DIR/$GSASL_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- ../configure --prefix=$TP_INSTALL_DIR --enable-shared=no --with-pic --with-libidn-prefix=$TP_INSTALL_DIR
- make -j $PARALLEL && make install
-}
-
-# build_gsasl2 just for libgsasl1.8.0
-build_gsasl2() {
- check_if_source_exist $GSASL2_SOURCE
- cd $TP_SOURCE_DIR/$GSASL2_SOURCE
- mkdir -p $BUILD_DIR && cd $BUILD_DIR
- ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --with-gssapi-impl=mit --enable-shared=no --with-pic --with-libidn-prefix=$TP_INSTALL_DIR
+ ../configure --prefix=$TP_INSTALL_DIR --with-gssapi-impl=mit --enable-shared=no --with-pic --with-libidn-prefix=$TP_INSTALL_DIR
make -j $PARALLEL && make install
}
@@ -956,29 +948,20 @@ build_krb5() {
check_if_source_exist $KRB5_SOURCE
cd $TP_SOURCE_DIR/$KRB5_SOURCE/src
mkdir -p $BUILD_DIR && cd $BUILD_DIR
- CFLAGS="-fcommon" \
- ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --disable-shared --enable-static
+ CFLAGS="-fcommon -fPIC" \
+ ../configure --prefix=$TP_INSTALL_DIR --disable-shared --enable-static
make -j $PARALLEL && make install
}
# hdfs3
build_hdfs3() {
- check_if_source_exist $HDFS3_SOURCE
- cd $TP_SOURCE_DIR/$HDFS3_SOURCE
- mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
- # build libhdfs3 without kerberos
- ../bootstrap --dependency="$TP_INSTALL_DIR" --prefix=$TP_INSTALL_DIR
- make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
- make install
-}
-
-# hdfs3_with_kerberos
-build_hdfs3_with_kerberos() {
check_if_source_exist $HDFS3_SOURCE
cd $TP_SOURCE_DIR/$HDFS3_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
# build libhdfs3 with kerberos support
- ../bootstrap --dependency="$HDFS3_KRB5_INSTALL_DIR:$TP_INSTALL_DIR -DWITH_KERBEROS=true" --prefix=$HDFS3_KRB5_INSTALL_DIR
+ CPPLAGS="-I${TP_INCLUDE_DIR} -fPIC" \
+ LDFLAGS="-L${TP_LIB_DIR}" \
+ ../bootstrap --dependency="$TP_INSTALL_DIR" --prefix=$TP_INSTALL_DIR --disable-shared --enable-static
make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
make install
}
@@ -1080,10 +1063,8 @@ build_lzma
build_xml2
build_idn
build_gsasl
-build_gsasl2
build_krb5
build_hdfs3
-build_hdfs3_with_kerberos
build_benchmark
build_simdjson
build_nlohmann_json
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 30ee5e7e55..60883f2632 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -273,22 +273,13 @@ echo "Finished patching $S2_SOURCE"
# gsasl2 patch to fix link error such as mutilple func defination
# when link target with kerberos
-cd $TP_SOURCE_DIR/$GSASL2_SOURCE
+cd $TP_SOURCE_DIR/$GSASL_SOURCE
if [ ! -f $PATCHED_MARK ]; then
patch -p1 < $TP_PATCH_DIR/libgsasl-1.8.0.patch
touch $PATCHED_MARK
fi
cd -
-echo "Finished patching $GSASL2_SOURCE"
-
-# hdfs3 patch to fix compile error
-cd $TP_SOURCE_DIR/$HDFS3_SOURCE
-if [ ! -f $PATCHED_MARK ]; then
- patch -p1 < $TP_PATCH_DIR/libhdfs3-master.patch
- touch $PATCHED_MARK
-fi
-cd -
-echo "Finished patching $HDFS3_SOURCE"
+echo "Finished patching $GSASL_SOURCE"
# rocksdb patch to fix compile error
if [ $ROCKSDB_SOURCE == "rocksdb-5.14.2" ]; then
diff --git a/thirdparty/patches/libhdfs3-master.patch b/thirdparty/patches/libhdfs3-master.patch
deleted file mode 100644
index 6c4eb2bfd7..0000000000
--- a/thirdparty/patches/libhdfs3-master.patch
+++ /dev/null
@@ -1,138 +0,0 @@
-diff -uprN a/src/CMakeLists.txt b/src/CMakeLists.txt
---- a/src/CMakeLists.txt 2021-09-23 22:03:55.000000000 +0800
-+++ b/src/CMakeLists.txt 2022-01-18 00:58:22.411061469 +0800
-@@ -46,7 +46,7 @@ SET(HEADER
- common/XmlConfig.h)
-
- ADD_LIBRARY(libhdfs3-static STATIC ${libhdfs3_SOURCES} ${libhdfs3_PROTO_SOURCES} ${libhdfs3_PROTO_HEADERS})
--ADD_LIBRARY(libhdfs3-shared SHARED ${libhdfs3_SOURCES} ${libhdfs3_PROTO_SOURCES} ${libhdfs3_PROTO_HEADERS})
-+ADD_LIBRARY(libhdfs3-shared STATIC ${libhdfs3_SOURCES} ${libhdfs3_PROTO_SOURCES} ${libhdfs3_PROTO_HEADERS})
-
- ADD_CUSTOM_COMMAND(
- TARGET libhdfs3-shared libhdfs3-static
-diff --git a/src/client/FileSystem.cpp b/src/client/FileSystem.cpp
-index 6c347c7..6aec1a3 100644
---- a/src/client/FileSystem.cpp
-+++ b/src/client/FileSystem.cpp
-@@ -136,7 +136,7 @@ static std::string ExtractPrincipalFromTicketCache(
- static std::string ExtractPrincipalFromToken(const Token & token) {
- std::string realUser, owner;
- std::string identifier = token.getIdentifier();
-- WritableUtils cin(identifier.data(), identifier.size());
-+ WritableUtils cin(&identifier[0], identifier.size());
- char version;
-
- try {
-diff --git a/src/client/Token.cpp b/src/client/Token.cpp
-index 1e23fed..8c88b2f 100644
---- a/src/client/Token.cpp
-+++ b/src/client/Token.cpp
-@@ -156,10 +156,10 @@ Token & Token::fromString(const std::string & str) {
- WritableUtils in(buffer.data(), buffer.size());
- len = in.ReadInt32();
- identifier.resize(len);
-- in.ReadRaw(identifier.data(), len);
-+ in.ReadRaw(&identifier[0], len);
- len = in.ReadInt32();
- password.resize(len);
-- in.ReadRaw(password.data(), len);
-+ in.ReadRaw(&password[0], len);
- kind = in.ReadText();
- service = in.ReadText();
- return *this;
-diff --git a/src/common/ExceptionInternal.h b/src/common/ExceptionInternal.h
-index 4ee661e..9d734af 100644
---- a/src/common/ExceptionInternal.h
-+++ b/src/common/ExceptionInternal.h
-@@ -175,7 +175,7 @@ void ThrowException(bool nested, const char * f, int l,
- int offset = buffer.size();
- buffer.resize(offset + size + 1);
- va_start(ap, fmt);
-- vsnprintf(buffer.data() + offset, size + 1, fmt, ap);
-+ vsnprintf(&buffer[offset], size + 1, fmt, ap);
- va_end(ap);
-
- if (!nested) {
-@@ -218,7 +218,7 @@ void ThrowException(bool nested, const char * f, int l,
- int offset = buffer.size();
- buffer.resize(offset + size + 1);
- va_start(ap, fmt);
-- vsnprintf(buffer.data() + offset, size + 1, fmt, ap);
-+ vsnprintf(&buffer[offset], size + 1, fmt, ap);
- va_end(ap);
-
- if (!nested) {
-diff --git a/src/common/StringUtil.h b/src/common/StringUtil.h
-index f9cba5d..3a8e76f 100644
---- a/src/common/StringUtil.h
-+++ b/src/common/StringUtil.h
-@@ -41,7 +41,7 @@ static inline std::vector<std::string> StringSplit(const std::string & str,
- char * token, *lasts = NULL;
- std::string s = str;
- std::vector<std::string> retval;
-- token = strtok_r(s.data(), sep, &lasts);
-+ token = strtok_r(&s[0], sep, &lasts);
-
- while (token) {
- retval.push_back(token);
-diff --git a/src/common/WritableUtils.cpp b/src/common/WritableUtils.cpp
-index fdadd45..98bbc6a 100644
---- a/src/common/WritableUtils.cpp
-+++ b/src/common/WritableUtils.cpp
-@@ -88,7 +88,7 @@ std::string WritableUtils::ReadText() {
- std::string retval;
- length = ReadInt32();
- retval.resize(length);
-- ReadRaw(retval.data(), length);
-+ ReadRaw(&retval[0], length);
- return retval;
- }
-
-diff --git a/src/rpc/RpcClient.cpp b/src/rpc/RpcClient.cpp
-index bb19991..ebb6e28 100644
---- a/src/rpc/RpcClient.cpp
-+++ b/src/rpc/RpcClient.cpp
-@@ -54,7 +54,7 @@ RpcClientImpl::RpcClientImpl() :
- cleaning(false), running(true), count(0) {
- auto id = boost::uuids::random_generator()();
- clientId.resize(boost::uuids::uuid::static_size());
-- memcpy(clientId.data(), id.begin(), boost::uuids::uuid::static_size());
-+ memcpy(&clientId[0], id.begin(), boost::uuids::uuid::static_size());
- #ifdef MOCK
- stub = NULL;
- #endif
-diff --git a/src/rpc/SaslClient.cpp b/src/rpc/SaslClient.cpp
-index 9aa9d7b..53893a1 100644
---- a/src/rpc/SaslClient.cpp
-+++ b/src/rpc/SaslClient.cpp
-@@ -137,7 +137,7 @@ std::string SaslClient::evaluateChallenge(const std::string & challenge) {
-
- if (rc == GSASL_NEEDS_MORE || rc == GSASL_OK) {
- retval.resize(outputSize);
-- memcpy(retval.data(), output, outputSize);
-+ memcpy(&retval[0], output, outputSize);
-
- if (output) {
- free(output);
-diff -uprN a/src/CMakeLists.txt b/src/CMakeLists.txt
---- a/bootstrap 2022-02-26 16:12:06.065389096 +0800
-+++ b/bootstrap 2022-02-26 16:11:45.989378097 +0800
-@@ -111,11 +111,17 @@ if [[ ! -x ${cmake} ]]; then
- die "cannot found cmake"
- fi
-
-+arch=$(uname -i)
-+if [[ $arch == arm* ]] || [[ $arch = aarch64 ]]; then
-+ CMAKE_EXTRA_FLAGS="-DENABLE_SSE=0"
-+fi
-+
- # Configure
- ${cmake} -DENABLE_DEBUG=${enable_build} -DCMAKE_INSTALL_PREFIX=${prefix_dirs} \
- -DCMAKE_C_COMPILER=${c_compiler} -DCMAKE_CXX_COMPILER=${cxx_compiler} \
- -DCMAKE_PREFIX_PATH=${dependency_dir} -DENABLE_BOOST=${enable_boost} \
- -DENABLE_COVERAGE=${enable_coverage} -DENABLE_LIBCPP=${enable_clang_lib} ${source_dir} \
-+ $CMAKE_EXTRA_FLAGS \
- || die "failed to configure the project"
-
- echo 'bootstrap success. Run "make" to build.'
-
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 1d975bf5cc..397c8649f3 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -32,9 +32,6 @@ export TP_SOURCE_DIR=$TP_DIR/src
# thirdparties will be installed to here
export TP_INSTALL_DIR=$TP_DIR/installed
-# libhdfs3-with-kerberos will be installed to here
-export HDFS3_KRB5_INSTALL_DIR=$TP_INSTALL_DIR/libhdfs_with_kerberos
-
# patches for all thirdparties
export TP_PATCH_DIR=$TP_DIR/patches
@@ -336,16 +333,10 @@ IDN_SOURCE="libidn-1.38"
IDN_MD5SUM="718ff3700dd71f830c592ebe97249193"
# gsasl
-GSASL_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.10.0.tar.gz"
-GSASL_NAME="libgsasl-1.10.0.tar.gz"
-GSASL_SOURCE="libgsasl-1.10.0"
-GSASL_MD5SUM="9c8fc632da4ce108fb7581b33de2a5ce"
-
-# gsasl 1.8.0 is only used for libhdfs3 with kerberos
-GSASL2_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz"
-GSASL2_NAME="libgsasl-1.8.0.tar.gz"
-GSASL2_SOURCE="libgsasl-1.8.0"
-GSASL2_MD5SUM="5dbdf859f6e60e05813370e2b193b92b"
+GSASL_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz"
+GSASL_NAME="libgsasl-1.8.0.tar.gz"
+GSASL_SOURCE="libgsasl-1.8.0"
+GSASL_MD5SUM="5dbdf859f6e60e05813370e2b193b92b"
# krb5
KRB5_DOWNLOAD="https://kerberos.org/dist/krb5/1.19/krb5-1.19.tar.gz"
@@ -354,10 +345,10 @@ KRB5_SOURCE="krb5-1.19"
KRB5_MD5SUM="aaf18447a5a014aa3b7e81814923f4c9"
# hdfs3
-HDFS3_DOWNLOAD="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/libhdfs3-master.zip"
-HDFS3_NAME="libhdfs3-master.zip"
-HDFS3_SOURCE="libhdfs3-master"
-HDFS3_MD5SUM="8c071fd2e7b0b1ccc1ec9c0d073d4146"
+HDFS3_DOWNLOAD="https://github.com/yangzhg/libhdfs3/archive/refs/tags/v2.3.0.tar.gz"
+HDFS3_NAME="libhdfs3-2.3.0.tar.gz"
+HDFS3_SOURCE="libhdfs3-2.3.0"
+HDFS3_MD5SUM="f647975fb7ad03bf25a14f530b1a5c06"
#libdivide
LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/5.0.tar.gz"
@@ -462,7 +453,6 @@ LZMA
XML2
IDN
GSASL
-GSASL2
KRB5
HDFS3
LIBDIVIDE
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org