You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2024/07/31 09:21:17 UTC
(doris-thirdparty) branch hadoop-3.4.0 updated (bd8b77f398f -> f98c08e2dcf)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
from bd8b77f398f HADOOP-19099. Add Protobuf Compatibility Notes (#6607) Contributed by Shilun Fan.
new be42d26e56c [feature] c api for use principal and keytab to login kerberos (#69)
new 1b4fde83eda [script] add build script (#71)
new 7389e5f640f [scripts] add native dir (#72)
new be310e5b125 [chore] skip java doc
new e18f529e522 [chore] Fix the workflow (#84)
new cf3f82bc554 [chore] Build HDFS libraries only (#85)
new 8a0fa7700be [chore] Build hadoop common native libs. (#98)
new 80e15055b46 [fix](tgt)hadoop kerberos support renew TGT by keytab (#173)
new f98c08e2dcf fix: do really detection of crc32 instructions (#233)
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.sh | 64 +++++++
dev-support/bin/dist-layout-stitching | 18 +-
.../lib/bindings/c/CMakeLists.txt => env.sh | 5 +-
hadoop-common-project/hadoop-common/pom.xml | 4 +
.../hadoop-common/src/CMakeLists.txt | 46 ++++-
.../main/java/org/apache/hadoop/fs/FileSystem.java | 13 ++
.../hadoop-hdfs-native-client/pom.xml | 14 ++
.../hadoop-hdfs-native-client/src/CMakeLists.txt | 43 +++--
.../src/main/native/libhdfs/exception.c | 10 +
.../src/main/native/libhdfs/hdfs.c | 203 ++++++++++++++++++---
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 18 ++
.../src/main/native/libhdfs/jclasses.c | 4 +
.../src/main/native/libhdfs/jclasses.h | 4 +
.../src/main/native/libhdfs/jni_helper.c | 20 +-
.../src/CMakeLists.txt | 82 +++++----
hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 30 +--
16 files changed, 469 insertions(+), 109 deletions(-)
create mode 100755 build.sh
copy hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt => env.sh (74%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 04/09: [chore] skip java doc
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit be310e5b125163a7beafeb64cd0f0081ba755b7a
Author: morningman <mo...@163.com>
AuthorDate: Fri May 26 09:31:56 2023 +0800
[chore] skip java doc
---
build.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/build.sh b/build.sh
index d06272f5ebe..8eb02ecb5b9 100755
--- a/build.sh
+++ b/build.sh
@@ -39,7 +39,7 @@ rm -rf ${DIST_DIR}
rm -rf ${LIBHDFS_DIST_DIR}
export PATH=${THIRDPARTY_INSTALLED}/bin:$PATH
-mvn clean package -Pnative,dist -DskipTests -Dthirdparty.installed=${THIRDPARTY_INSTALLED}/ -Dopenssl.lib=${THIRDPARTY_INSTALLED}/lib/ -e
+mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip=true -Dthirdparty.installed=${THIRDPARTY_INSTALLED}/ -Dopenssl.lib=${THIRDPARTY_INSTALLED}/lib/ -e
if [[ ! -d "${DIST_DIR}" ]]; then
echo "${DIST_DIR} is missing. Build failed."
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 07/09: [chore] Build hadoop common native libs. (#98)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit 8a0fa7700be98cbafdd07a3c71e7a318f4ce58bf
Author: Qi Chen <ka...@gmail.com>
AuthorDate: Fri Jul 21 11:50:12 2023 +0800
[chore] Build hadoop common native libs. (#98)
---
build.sh | 3 ++-
hadoop-common-project/hadoop-common/pom.xml | 4 ++++
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/build.sh b/build.sh
index 0cc81fbe3d5..502a7123ce4 100755
--- a/build.sh
+++ b/build.sh
@@ -38,7 +38,8 @@ LIBHDFS_DIST_DIR="${SRC_HOME}/hadoop-dist/target/hadoop-libhdfs-3.3.6"
rm -rf "${DIST_DIR}"
rm -rf "${LIBHDFS_DIST_DIR}"
-mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-common-project -e
+mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip -f hadoop-common-project \
+ -Dopenssl.prefix="${THIRDPARTY_INSTALLED}" -e
mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip -f hadoop-hdfs-project \
-Dthirdparty.installed="${THIRDPARTY_INSTALLED}" -Dopenssl.prefix="${THIRDPARTY_INSTALLED}" -e
mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-dist -e
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 66c07cfee7c..4223499cbc0 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -766,6 +766,10 @@
<goals><goal>cmake-compile</goal></goals>
<configuration>
<source>${basedir}/src</source>
+ <env>
+ <CFLAGS>-Wno-implicit-function-declaration</CFLAGS>
+ <CXXFLAGS>-Wno-implicit-function-declaration</CXXFLAGS>
+ </env>
<vars>
<GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH>
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 03/09: [scripts] add native dir (#72)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit 7389e5f640fd2fe0516e2e7cadf2648b563f3927
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Thu May 18 21:13:07 2023 +0800
[scripts] add native dir (#72)
---
build.sh | 2 ++
1 file changed, 2 insertions(+)
diff --git a/build.sh b/build.sh
index 342f2af5bd0..d06272f5ebe 100755
--- a/build.sh
+++ b/build.sh
@@ -50,9 +50,11 @@ echo "Finished. Begin to pacakge for libhdfs..."
mkdir -p ${LIBHDFS_DIST_DIR}/common
mkdir -p ${LIBHDFS_DIST_DIR}/hdfs
mkdir -p ${LIBHDFS_DIST_DIR}/include
+mkdir -p ${LIBHDFS_DIST_DIR}/native
cp -r ${DIST_DIR}/share/hadoop/common/* ${LIBHDFS_DIST_DIR}/common/
cp -r ${DIST_DIR}/share/hadoop/hdfs/* ${LIBHDFS_DIST_DIR}/hdfs/
cp -r ${DIST_DIR}/include/hdfs.h ${LIBHDFS_DIST_DIR}/include/
+cp -r ${DIST_DIR}/lib/native/libhdfs.a ${LIBHDFS_DIST_DIR}/native/
echo "Done!"
echo "The full dist package is under: ${DIST_DIR}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 05/09: [chore] Fix the workflow (#84)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit e18f529e522fcd14354cefefc7bb9d95efc3ac12
Author: Adonis Ling <ad...@gmail.com>
AuthorDate: Tue May 30 22:06:50 2023 +0800
[chore] Fix the workflow (#84)
Fix lots of issues with GitHub workflows.
---
build.sh | 36 +++++-----
.../hadoop-hdfs-native-client/src/CMakeLists.txt | 43 ++++++++----
.../src/CMakeLists.txt | 82 ++++++++++++----------
hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 30 ++++----
4 files changed, 109 insertions(+), 82 deletions(-)
diff --git a/build.sh b/build.sh
index 8eb02ecb5b9..717388f3eaf 100755
--- a/build.sh
+++ b/build.sh
@@ -29,34 +29,34 @@ export SRC_HOME="${ROOT}"
. "${SRC_HOME}/env.sh"
if [[ -z "${THIRDPARTY_INSTALLED}" ]]; then
- echo "Must set 'THIRDPARTY_INSTALLED' in env.sh"
- exit -1
+ echo "Must set 'THIRDPARTY_INSTALLED' in env.sh"
+ exit 255
fi
-DIST_DIR=${SRC_HOME}/hadoop-dist/target/hadoop-3.3.4/
-LIBHDFS_DIST_DIR=${SRC_HOME}/hadoop-dist/target/hadoop-libhdfs-3.3.4/
-rm -rf ${DIST_DIR}
-rm -rf ${LIBHDFS_DIST_DIR}
+DIST_DIR="${SRC_HOME}/hadoop-dist/target/hadoop-3.3.6"
+LIBHDFS_DIST_DIR="${SRC_HOME}/hadoop-dist/target/hadoop-libhdfs-3.3.6"
+rm -rf "${DIST_DIR}"
+rm -rf "${LIBHDFS_DIST_DIR}"
export PATH=${THIRDPARTY_INSTALLED}/bin:$PATH
-mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip=true -Dthirdparty.installed=${THIRDPARTY_INSTALLED}/ -Dopenssl.lib=${THIRDPARTY_INSTALLED}/lib/ -e
+mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip=true \
+ -Dthirdparty.installed="${THIRDPARTY_INSTALLED}" -Dopenssl.prefix="${THIRDPARTY_INSTALLED}" -e
if [[ ! -d "${DIST_DIR}" ]]; then
- echo "${DIST_DIR} is missing. Build failed."
- exit -1
+ echo "${DIST_DIR} is missing. Build failed."
+ exit 255
fi
echo "Finished. Begin to pacakge for libhdfs..."
-mkdir -p ${LIBHDFS_DIST_DIR}/common
-mkdir -p ${LIBHDFS_DIST_DIR}/hdfs
-mkdir -p ${LIBHDFS_DIST_DIR}/include
-mkdir -p ${LIBHDFS_DIST_DIR}/native
-cp -r ${DIST_DIR}/share/hadoop/common/* ${LIBHDFS_DIST_DIR}/common/
-cp -r ${DIST_DIR}/share/hadoop/hdfs/* ${LIBHDFS_DIST_DIR}/hdfs/
-cp -r ${DIST_DIR}/include/hdfs.h ${LIBHDFS_DIST_DIR}/include/
-cp -r ${DIST_DIR}/lib/native/libhdfs.a ${LIBHDFS_DIST_DIR}/native/
+mkdir -p "${LIBHDFS_DIST_DIR}/common"
+mkdir -p "${LIBHDFS_DIST_DIR}/hdfs"
+mkdir -p "${LIBHDFS_DIST_DIR}/include"
+mkdir -p "${LIBHDFS_DIST_DIR}/native"
+cp -r "${DIST_DIR}/share/hadoop/common"/* "${LIBHDFS_DIST_DIR}/common"/
+cp -r "${DIST_DIR}/share/hadoop/hdfs"/* "${LIBHDFS_DIST_DIR}/hdfs"/
+cp -r "${DIST_DIR}/include/hdfs.h" "${LIBHDFS_DIST_DIR}/include"/
+cp -r "${DIST_DIR}/lib/native/libhdfs.a" "${LIBHDFS_DIST_DIR}/native"/
echo "Done!"
echo "The full dist package is under: ${DIST_DIR}"
echo "The LIBHDFS dist package is under: ${LIBHDFS_DIST_DIR}"
-
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index b44109f3092..06eb916bcaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -26,6 +26,9 @@ cmake_policy(SET CMP0079 NEW)
project(hadoop_hdfs_native_client)
+option(CHANGE_FIND_LIBRARY_SUFFIXES "Change suffixes to find library" OFF)
+option(LIBRARY_ONLY "Build libraries only" ON)
+
enable_testing()
set(CMAKE_CXX_STANDARD 17)
@@ -104,29 +107,37 @@ endif()
include(HadoopJNI)
function(build_libhdfs_test NAME LIBRARY)
- set(FILES)
- foreach(FIL ${ARGN})
- if (IS_ABSOLUTE ${FIL})
- list(APPEND FILES ${FIL})
- else()
- list(APPEND FILES ${CMAKE_SOURCE_DIR}/main/native/libhdfs-tests/${FIL})
- endif()
- endforeach()
- add_executable("${NAME}_${LIBRARY}" $<TARGET_OBJECTS:x_platform_obj_c_api> $<TARGET_OBJECTS:x_platform_obj> ${FILES})
- target_include_directories("${NAME}_${LIBRARY}" PRIVATE main/native/libhdfspp/lib)
+ if (NOT LIBRARY_ONLY)
+ set(FILES)
+ foreach(FIL ${ARGN})
+ if (IS_ABSOLUTE ${FIL})
+ list(APPEND FILES ${FIL})
+ else()
+ list(APPEND FILES ${CMAKE_SOURCE_DIR}/main/native/libhdfs-tests/${FIL})
+ endif()
+ endforeach()
+ add_executable("${NAME}_${LIBRARY}" $<TARGET_OBJECTS:x_platform_obj_c_api> $<TARGET_OBJECTS:x_platform_obj> ${FILES})
+ target_include_directories("${NAME}_${LIBRARY}" PRIVATE main/native/libhdfspp/lib)
+ endif()
endfunction()
function(add_libhdfs_test NAME LIBRARY)
- add_test("test_${NAME}_${LIBRARY}" "${NAME}_${LIBRARY}")
+ if (NOT LIBRARY_ONLY)
+ add_test("test_${NAME}_${LIBRARY}" "${NAME}_${LIBRARY}")
+ endif()
endfunction()
function(link_libhdfs_test NAME LIBRARY)
-target_link_libraries("${NAME}_${LIBRARY}" ${LIBRARY} ${ARGN})
+ if (NOT LIBRARY_ONLY)
+ target_link_libraries("${NAME}_${LIBRARY}" ${LIBRARY} ${ARGN})
+ endif()
endfunction()
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-hadoop_set_find_shared_library_without_version()
+if (CHANGE_FIND_LIBRARY_SUFFIXES)
+ hadoop_set_find_shared_library_without_version()
+endif()
set(OPENSSL_NAME "crypto")
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
SET(OPENSSL_NAME "libcrypto")
@@ -174,8 +185,10 @@ else()
endif()
add_subdirectory(main/native/libhdfs)
-add_subdirectory(main/native/libhdfs-tests)
-add_subdirectory(main/native/libhdfs-examples)
+if (NOT LIBRARY_ONLY)
+ add_subdirectory(main/native/libhdfs-tests)
+ add_subdirectory(main/native/libhdfs-examples)
+endif()
# Temporary fix to disable Libhdfs++ build on older systems that do not support thread_local
include(CheckCXXSourceCompiles)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index b9a011ef821..1a0384d50fd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -18,6 +18,8 @@
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
+option(LIBRARY_ONLY "Build libraries only" ON)
+
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-common/)
include(HadoopCommon)
@@ -27,6 +29,9 @@ set(GTEST_SRC_DIR ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-c
# Add extra compiler and linker flags.
# -Wno-sign-compare
hadoop_add_compiler_flags("-DNDEBUG -DSIMPLE_MEMCPY -fno-strict-aliasing -fsigned-char")
+if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+ hadoop_add_compiler_flags(-Wno-reserved-user-defined-literal)
+endif()
set(CMAKE_CXX_STANDARD 11)
# Source location.
@@ -161,41 +166,43 @@ hadoop_add_dual_library(nativetask
target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
-add_library(gtest ${GTEST_SRC_DIR}/gtest-all.cc)
-set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
-add_executable(nttest
- ${SRC}/test/lib/TestByteArray.cc
- ${SRC}/test/lib/TestByteBuffer.cc
- ${SRC}/test/lib/TestComparatorForDualPivotQuickSort.cc
- ${SRC}/test/lib/TestComparatorForStdSort.cc
- ${SRC}/test/lib/TestFixSizeContainer.cc
- ${SRC}/test/lib/TestMemoryPool.cc
- ${SRC}/test/lib/TestIterator.cc
- ${SRC}/test/lib/TestKVBuffer.cc
- ${SRC}/test/lib/TestMemBlockIterator.cc
- ${SRC}/test/lib/TestMemoryBlock.cc
- ${SRC}/test/lib/TestPartitionBucket.cc
- ${SRC}/test/lib/TestReadBuffer.cc
- ${SRC}/test/lib/TestReadWriteBuffer.cc
- ${SRC}/test/util/TestChecksum.cc
- ${SRC}/test/util/TestStringUtil.cc
- ${SRC}/test/util/TestWritableUtils.cc
- ${SRC}/test/TestCommand.cc
- ${SRC}/test/TestConfig.cc
- ${SRC}/test/TestCounter.cc
- ${SRC}/test/TestCompressions.cc
- ${SRC}/test/TestFileSystem.cc
- ${SRC}/test/TestIFile.cc
- ${SRC}/test/TestPrimitives.cc
- ${SRC}/test/TestSort.cc
- ${SRC}/test/TestMain.cc
- ${SRC}/test/test_commons.cc)
-
-target_link_libraries(nttest
- nativetask_static
- gtest
- ${NT_DEPEND_LIBRARY}
-)
+if (NOT LIBRARY_ONLY)
+ add_library(gtest ${GTEST_SRC_DIR}/gtest-all.cc)
+ set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
+ add_executable(nttest
+ ${SRC}/test/lib/TestByteArray.cc
+ ${SRC}/test/lib/TestByteBuffer.cc
+ ${SRC}/test/lib/TestComparatorForDualPivotQuickSort.cc
+ ${SRC}/test/lib/TestComparatorForStdSort.cc
+ ${SRC}/test/lib/TestFixSizeContainer.cc
+ ${SRC}/test/lib/TestMemoryPool.cc
+ ${SRC}/test/lib/TestIterator.cc
+ ${SRC}/test/lib/TestKVBuffer.cc
+ ${SRC}/test/lib/TestMemBlockIterator.cc
+ ${SRC}/test/lib/TestMemoryBlock.cc
+ ${SRC}/test/lib/TestPartitionBucket.cc
+ ${SRC}/test/lib/TestReadBuffer.cc
+ ${SRC}/test/lib/TestReadWriteBuffer.cc
+ ${SRC}/test/util/TestChecksum.cc
+ ${SRC}/test/util/TestStringUtil.cc
+ ${SRC}/test/util/TestWritableUtils.cc
+ ${SRC}/test/TestCommand.cc
+ ${SRC}/test/TestConfig.cc
+ ${SRC}/test/TestCounter.cc
+ ${SRC}/test/TestCompressions.cc
+ ${SRC}/test/TestFileSystem.cc
+ ${SRC}/test/TestIFile.cc
+ ${SRC}/test/TestPrimitives.cc
+ ${SRC}/test/TestSort.cc
+ ${SRC}/test/TestMain.cc
+ ${SRC}/test/test_commons.cc)
+
+ target_link_libraries(nttest
+ nativetask_static
+ gtest
+ ${NT_DEPEND_LIBRARY}
+ )
+endif()
# By embedding '$ORIGIN' into the RPATH of libnativetask.so, dlopen will look in
# the directory containing libnativetask.so. However, $ORIGIN is not supported by
@@ -207,4 +214,7 @@ endif()
set(LIBNATIVETASK_VERSION "1.0.0")
set_target_properties(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION})
hadoop_dual_output_directory(nativetask target/usr/local/lib)
-hadoop_output_directory(nttest test)
+
+if (NOT LIBRARY_ONLY)
+ hadoop_output_directory(nttest test)
+endif()
diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
index ce6ee317936..88180795f17 100644
--- a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
+++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
@@ -18,6 +18,8 @@
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
+option(LIBRARY_ONLY "Build libraries only" ON)
+
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
include(HadoopCommon)
@@ -49,22 +51,24 @@ include_directories(
${OPENSSL_INCLUDE_DIR}
)
-# Example programs
-add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
-target_link_libraries(wordcount-simple hadooppipes hadooputils)
-hadoop_output_directory(wordcount-simple examples)
+if (NOT LIBRARY_ONLY)
+ # Example programs
+ add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
+ target_link_libraries(wordcount-simple hadooppipes hadooputils)
+ hadoop_output_directory(wordcount-simple examples)
-add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
-target_link_libraries(wordcount-part hadooppipes hadooputils)
-hadoop_output_directory(wordcount-part examples)
+ add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
+ target_link_libraries(wordcount-part hadooppipes hadooputils)
+ hadoop_output_directory(wordcount-part examples)
-add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
-target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
-hadoop_output_directory(wordcount-nopipe examples)
+ add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
+ target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
+ hadoop_output_directory(wordcount-nopipe examples)
-add_executable(pipes-sort main/native/examples/impl/sort.cc)
-target_link_libraries(pipes-sort hadooppipes hadooputils)
-hadoop_output_directory(pipes-sort examples)
+ add_executable(pipes-sort main/native/examples/impl/sort.cc)
+ target_link_libraries(pipes-sort hadooppipes hadooputils)
+ hadoop_output_directory(pipes-sort examples)
+endif()
add_library(hadooputils STATIC
main/native/utils/impl/StringUtils.cc
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 01/09: [feature] c api for use principal and keytab to login kerberos (#69)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit be42d26e56ce170acf344fa6ff98808a44d04161
Author: Long Zhao <29...@qq.com>
AuthorDate: Thu May 18 11:18:55 2023 +0800
[feature] c api for use principal and keytab to login kerberos (#69)
---
.../src/main/native/libhdfs/exception.c | 10 ++
.../src/main/native/libhdfs/hdfs.c | 143 +++++++++++++++++++--
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 9 ++
.../src/main/native/libhdfs/jclasses.c | 4 +
.../src/main/native/libhdfs/jclasses.h | 4 +
.../src/main/native/libhdfs/jni_helper.c | 18 ++-
6 files changed, 168 insertions(+), 20 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
index fec9a103b4e..ed61b77fcad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
@@ -90,6 +90,16 @@ static const struct ExceptionInfo gExceptionInfo[] = {
0,
ESTALE,
},
+ {
+ "org.apache.hadoop.security.KerberosAuthException",
+ NOPRINT_EXC_ACCESS_CONTROL,
+ ETIME
+ },
+ {
+ "java.net.ConnectException",
+ 0,
+ EFAULT
+ },
};
void getExceptionInfo(const char *excName, int noPrintFlags,
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index ed150925cdb..ea00393cfc7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -25,6 +25,7 @@
#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
+#include <stdlib.h>
#include <string.h>
#define JAVA_VOID "V"
@@ -495,6 +496,11 @@ done:
return ret;
}
+struct hdfsBuilderConfFileOpt {
+ struct hdfsBuilderConfFileOpt *next;
+ const char *currentPath;
+};
+
struct hdfsBuilderConfOpt {
struct hdfsBuilderConfOpt *next;
const char *key;
@@ -506,10 +512,26 @@ struct hdfsBuilder {
const char *nn;
tPort port;
const char *kerbTicketCachePath;
+ const char *kerb5ConfPath;
+ const char *keyTabFile;
const char *userName;
struct hdfsBuilderConfOpt *opts;
+ struct hdfsBuilderConfFileOpt *fileOpts;
};
+void hdfsBuilderSetKerb5Conf(struct hdfsBuilder *bld, const char *kerb5ConfPath) {
+ bld->kerb5ConfPath = kerb5ConfPath;
+ if (bld->kerb5ConfPath) {
+ systemPropertySetStr("java.security.krb5.conf", bld->kerb5ConfPath);
+ hdfsBuilderConfSetStr(bld, "hadoop.security.authorization", "true");
+ hdfsBuilderConfSetStr(bld, "hadoop.security.authentication", "kerberos");
+ }
+}
+
+void hdfsBuilderSetKeyTabFile(struct hdfsBuilder *bld, const char *keyTabFile) {
+ bld->keyTabFile = keyTabFile;
+}
+
struct hdfsBuilder *hdfsNewBuilder(void)
{
struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder));
@@ -520,6 +542,42 @@ struct hdfsBuilder *hdfsNewBuilder(void)
return bld;
}
+int systemPropertySetStr(const char *key, const char *val) {
+ JNIEnv *env = 0;
+ jobject jRet = NULL;
+ jvalue jVal;
+ jthrowable jthr;
+ int ret = EINTERNAL;
+ jstring jkey = NULL, jvalue0 = NULL;
+ env = getJNIEnv();
+ if (env == NULL) {
+ ret = EINTERNAL;
+ return ret;
+ }
+ jthr = newJavaStr(env, key, &jkey);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"System::setProperty::new String");
+ goto done;
+ }
+ jthr = newJavaStr(env, val, &jvalue0);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"System::setProperty::new String");
+ goto done;
+ }
+ jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_SYSTEM, "setProperty", "(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;", jkey, jvalue0);
+ jRet = jVal.l;
+ ret = 0;
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"System::setProperty");
+ goto done;
+ }
+ done:
+ destroyLocalReference(env, jRet);
+ destroyLocalReference(env, jkey);
+ destroyLocalReference(env, jvalue0);
+ return ret;
+}
+
int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key,
const char *val)
{
@@ -546,6 +604,13 @@ void hdfsFreeBuilder(struct hdfsBuilder *bld)
free(cur);
cur = next;
}
+ struct hdfsBuilderConfFileOpt *cur0, *next0;
+ cur0 = bld->fileOpts;
+ for (cur0 = bld->fileOpts; cur0;) {
+ next0 = cur0->next;
+ free(cur0);
+ cur0 = next0;
+ }
free(bld);
}
@@ -688,17 +753,32 @@ static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld,
return buf;
}
+jthrowable hadoopConfFileAdd(JNIEnv *env, jobject jConfiguration, const char *path) {
+ /* Create an object of org.apache.hadoop.fs.Path */
+ jobject jPath = NULL;
+ jthrowable jthr;
+ jthr = constructNewObjectOfPath(env, path, &jPath);
+ if (jthr) {
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, JC_CONFIGURATION, "addResource", JMETHOD1(JPARAM(HADOOP_PATH), JAVA_VOID), jPath);
+ done:
+ destroyLocalReference(env, jPath);
+ return jthr;
+}
+
hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
{
JNIEnv *env = 0;
jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL;
- jstring jURIString = NULL, jUserString = NULL;
+ jstring jURIString = NULL, jUserString = NULL, jKeyTabString = NULL;
jvalue jVal;
jthrowable jthr = NULL;
char *cURI = 0, buf[512];
int ret;
jobject jRet = NULL;
struct hdfsBuilderConfOpt *opt;
+ struct hdfsBuilderConfFileOpt *fileOpt;
//Get the JNIEnv* corresponding to current thread
env = getJNIEnv();
@@ -715,6 +795,18 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
goto done;
}
+
+ for (fileOpt = bld->fileOpts; fileOpt; fileOpt = fileOpt->next) {
+ // conf.addResource(new Path(fileOpt->path))
+ jthr = hadoopConfFileAdd(env, jConfiguration, fileOpt->currentPath);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s): error adding conffile '%s'",
+ hdfsBuilderToStr(bld, buf, sizeof(buf)), fileOpt->currentPath);
+ goto done;
+ }
+ }
+
// set configuration values
for (opt = bld->opts; opt; opt = opt->next) {
jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val);
@@ -794,7 +886,30 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
jURI = jVal.l;
}
- if (bld->kerbTicketCachePath) {
+ jthr = newJavaStr(env, bld->userName, &jUserString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ if (bld->kerb5ConfPath && bld->keyTabFile) {
+ jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "setConfiguration", JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), jUserString, jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ } else if (bld->kerbTicketCachePath) {
jthr = hadoopConfSetStr(env, jConfiguration,
KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath);
if (jthr) {
@@ -803,13 +918,8 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
hdfsBuilderToStr(bld, buf, sizeof(buf)));
goto done;
}
- }
- jthr = newJavaStr(env, bld->userName, &jUserString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
- "hdfsBuilderConnect(%s)",
- hdfsBuilderToStr(bld, buf, sizeof(buf)));
- goto done;
+ invokeMethod(env, &jVal, STATIC, NULL, JC_SECURITY_CONFIGURATION, "setConfiguration",
+ JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
}
if (bld->forceNewInstance) {
jthr = invokeMethod(env, &jVal, STATIC, NULL,
@@ -825,11 +935,15 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
}
jFS = jVal.l;
} else {
- jthr = invokeMethod(env, &jVal, STATIC, NULL,
- JC_FILE_SYSTEM, "get",
- JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
- JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI,
- jConfiguration, jUserString);
+ if (bld->keyTabFile && bld->kerb5ConfPath) {
+ jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM, "get", JMETHOD1(JPARAM(HADOOP_CONF),
+ JPARAM(HADOOP_FS)), jConfiguration);
+ } else {
+ jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM, "get",
+ JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
+ JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)),
+ jURI, jConfiguration, jUserString);
+ }
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
"hdfsBuilderConnect(%s)",
@@ -856,6 +970,7 @@ done:
destroyLocalReference(env, jCachePath);
destroyLocalReference(env, jURIString);
destroyLocalReference(env, jUserString);
+ destroyLocalReference(env, jKeyTabString);
free(cURI);
hdfsFreeBuilder(bld);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
index eba50ff6eb2..ac2de2d7467 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
@@ -294,6 +294,9 @@ extern "C" {
LIBHDFS_EXTERNAL
struct hdfsBuilder *hdfsNewBuilder(void);
+ LIBHDFS_EXTERNAL
+ int systemPropertySetStr(const char *key, const char *val);
+
/**
* Force the builder to always create a new instance of the FileSystem,
* rather than possibly finding one in the cache.
@@ -357,6 +360,12 @@ extern "C" {
void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld,
const char *kerbTicketCachePath);
+ LIBHDFS_EXTERNAL
+ void hdfsBuilderSetKerb5Conf(struct hdfsBuilder *bld, const char *kerb5ConfPath);
+
+ LIBHDFS_EXTERNAL
+ void hdfsBuilderSetKeyTabFile(struct hdfsBuilder *bld, const char *keyTabFile);
+
/**
* Free an HDFS builder.
*
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
index 9f589ac257a..44acb08d27f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.c
@@ -112,6 +112,10 @@ jthrowable initCachedClasses(JNIEnv* env) {
"org/apache/commons/lang3/exception/ExceptionUtils";
cachedJavaClasses[JC_CFUTURE].className =
"java/util/concurrent/CompletableFuture";
+ cachedJavaClasses[JC_SYSTEM].className =
+ "java/lang/System";
+ cachedJavaClasses[JC_SECURITY_CONFIGURATION].className =
+ "org/apache/hadoop/security/UserGroupInformation";
// Create and set the jclass objects based on the class names set above
jthrowable jthr;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
index 0b174e1fecc..4f28ef2c027 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jclasses.h
@@ -61,6 +61,8 @@ typedef enum {
JC_ENUM_SET,
JC_EXCEPTION_UTILS,
JC_CFUTURE,
+ JC_SYSTEM,
+ JC_SECURITY_CONFIGURATION,
// A special marker enum that counts the number of cached jclasses
NUM_CACHED_CLASSES
} CachedJavaClass;
@@ -101,6 +103,7 @@ const char *getClassName(CachedJavaClass cachedJavaClass);
#define HADOOP_FS_BLDR "org/apache/hadoop/fs/FSBuilder"
#define HADOOP_RO "org/apache/hadoop/fs/ReadOption"
#define HADOOP_DS "org/apache/hadoop/net/unix/DomainSocket"
+#define HADOOP_USER_INFORMATION "org/apache/hadoop/security/UserGroupInformation"
/* Some frequently used Java class names */
#define JAVA_NET_ISA "java/net/InetSocketAddress"
@@ -111,6 +114,7 @@ const char *getClassName(CachedJavaClass cachedJavaClass);
#define JAVA_CFUTURE "java/util/concurrent/CompletableFuture"
#define JAVA_TIMEUNIT "java/util/concurrent/TimeUnit"
#define JAVA_OBJECT "java/lang/Object"
+#define JAVA_SYSTEM "java/lang/System"
/* Some frequently used third-party class names */
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
index 8f00a08b0a9..f1f7715e0e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
@@ -656,7 +656,7 @@ static char* getClassPath()
static JNIEnv* getGlobalJNIEnv(void)
{
JavaVM* vmBuf[VM_BUF_LENGTH];
- JNIEnv *env;
+ JNIEnv *env = NULL;
jint rv = 0;
jint noVMs = 0;
jthrowable jthr;
@@ -816,12 +816,9 @@ JNIEnv* getJNIEnv(void)
if (!state) {
mutexUnlock(&jvmMutex);
fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n");
- return NULL;
- }
- if (threadLocalStorageSet(state)) {
- mutexUnlock(&jvmMutex);
goto fail;
}
+ state->env = NULL;
THREAD_LOCAL_STORAGE_SET_QUICK(state);
state->env = getGlobalJNIEnv();
@@ -831,6 +828,12 @@ JNIEnv* getJNIEnv(void)
goto fail;
}
+ // when the env is created, the state can be set.
+ if (threadLocalStorageSet(state)) {
+ mutexUnlock(&jvmMutex);
+ goto fail;
+ }
+
jthrowable jthr = NULL;
jthr = initCachedClasses(state->env);
if (jthr) {
@@ -842,7 +845,10 @@ JNIEnv* getJNIEnv(void)
fail:
fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n");
- hdfsThreadDestructor(state);
+ THREAD_LOCAL_STORAGE_SET_QUICK(NULL);
+ if (state) {
+ hdfsThreadDestructor(state);
+ }
return NULL;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 09/09: fix: do really detection of crc32 instructions (#233)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit f98c08e2dcf4a3beb766e4e337865382bbb7ac16
Author: zclllhhjj <zh...@selectdb.com>
AuthorDate: Fri Jul 26 10:11:42 2024 +0800
fix: do really detection of crc32 instructions (#233)
before, in arm which before arm-v8 or x86 which before sse4.2, we will get wrong result of crc32 support. it will lead to compiliation error.
---
.../hadoop-common/src/CMakeLists.txt | 46 ++++++++++++++++++++--
1 file changed, 43 insertions(+), 3 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
index d2ef03645a4..d8792dcb592 100644
--- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt
+++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt
@@ -22,6 +22,8 @@
cmake_minimum_required(VERSION 3.1 FATAL_ERROR)
+include(CheckCSourceRuns)
+
list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/..)
include(HadoopCommon)
@@ -152,14 +154,52 @@ else(REQUIRE_PMDK)
MESSAGE(STATUS "Build without PMDK support.")
endif(REQUIRE_PMDK)
-# Build hardware CRC32 acceleration, if supported on the platform.
+# Build hardware CRC32 acceleration, if supported on the platform. We must detect the instructions. not all platform support relative instruction.
+
+# CMAKE_REQUIRED_FLAGS effects check_c_source_runs. temporarily detect with native architecture
+SET(TMP_CMAKE_FLAGS "${CMAKE_REQUIRED_FLAGS}")
+SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -march=native")
+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
- set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_x86.c")
+# x86, supported by sse4.2
+ check_c_source_runs("
+ #include <nmmintrin.h>
+ int main() {
+ unsigned int crc = 0;
+ unsigned char data[4] = {0x12, 0x34, 0x56, 0x78};
+ crc = _mm_crc32_u32(crc, *((unsigned int*)data));
+ return 0;
+ }" CRC_TEST_RESULT)
+ message(STATUS "SUPPORT OF CRC: ${CRC_TEST_RESULT}")
+ if (CRC_TEST_RESULT)
+ set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_x86.c")
+ else()
+ message("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW")
+ endif()
+
elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
- set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_aarch64.c")
+# arm64, supported by arm-v8
+ check_c_source_runs("
+ #include <arm_acle.h>
+ int main() {
+ uint32_t crc = 0;
+ uint32_t data = 12345;
+ crc = __crc32cw(crc, data);
+ return 0;
+ }" CRC_TEST_RESULT)
+ message(STATUS "SUPPORT OF CRC: ${CRC_TEST_RESULT}")
+ if (CRC_TEST_RESULT)
+ set(BULK_CRC_ARCH_SOURCE_FIlE "${SRC}/util/bulk_crc32_aarch64.c")
+ else()
+ message("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW")
+ endif()
+
else()
message("No HW CRC acceleration for ${CMAKE_SYSTEM_PROCESSOR}, falling back to SW")
endif()
+# revert the flags
+SET(CMAKE_REQUIRED_FLAGS "${TMP_CMAKE_FLAGS}")
+UNSET(TMP_CMAKE_FLAGS)
# Find the no-suffix version of libcrypto/openssl. See HADOOP-11216 for details.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 06/09: [chore] Build HDFS libraries only (#85)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit cf3f82bc5548625480dead2b8f8d54b38ed7b11a
Author: Adonis Ling <ad...@gmail.com>
AuthorDate: Fri Jun 2 13:19:33 2023 +0800
[chore] Build HDFS libraries only (#85)
1. We don't need to build the whole hadoop project. We can build HDFS related libraries to save the build time.
2. Port to GCC-13
---
build.sh | 5 +++--
dev-support/bin/dist-layout-stitching | 18 ++++++++++--------
hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 16 ++++++++++++++--
.../src/main/native/libhdfspp/CMakeLists.txt | 5 -----
.../src/CMakeLists.txt | 1 -
5 files changed, 27 insertions(+), 18 deletions(-)
diff --git a/build.sh b/build.sh
index 717388f3eaf..0cc81fbe3d5 100755
--- a/build.sh
+++ b/build.sh
@@ -38,9 +38,10 @@ LIBHDFS_DIST_DIR="${SRC_HOME}/hadoop-dist/target/hadoop-libhdfs-3.3.6"
rm -rf "${DIST_DIR}"
rm -rf "${LIBHDFS_DIST_DIR}"
-export PATH=${THIRDPARTY_INSTALLED}/bin:$PATH
-mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip=true \
+mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-common-project -e
+mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip -f hadoop-hdfs-project \
-Dthirdparty.installed="${THIRDPARTY_INSTALLED}" -Dopenssl.prefix="${THIRDPARTY_INSTALLED}" -e
+mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-dist -e
if [[ ! -d "${DIST_DIR}" ]]; then
echo "${DIST_DIR} is missing. Build failed."
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index d4bfd8aaada..4d63d18b238 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -142,14 +142,16 @@ run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" .
run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* .
run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* .
-# copy client jars as-is
-run mkdir -p "share/hadoop/client"
-run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
-run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
-run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
-
-run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
-run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" .
+if ! ${SKIP_HADOOP_CLIENT_AND_TOOLS:-true}; then
+ # copy client jars as-is
+ run mkdir -p "share/hadoop/client"
+ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
+ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
+ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
+
+ run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
+ run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" .
+fi
echo
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index e9be65d4938..3e7589bae1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -226,6 +226,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<goals><goal>cmake-compile</goal></goals>
<configuration>
<source>${basedir}/src</source>
+ <env>
+ <CXXFLAGS>-include cstdint</CXXFLAGS>
+ <PROTOBUF_HOME>${thirdparty.installed}</PROTOBUF_HOME>
+ </env>
<vars>
<GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH>
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
@@ -237,7 +241,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX>
<CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB>
<CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE>
- <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED>
+ <CUSTOM_SNAPPY_PREFIX>${thirdparty.installed}</CUSTOM_SNAPPY_PREFIX>
+ <HDFSPP_LIBRARY_ONLY>ON</HDFSPP_LIBRARY_ONLY>
+ <Boost_USE_STATIC_RUNTIME>ON</Boost_USE_STATIC_RUNTIME>
</vars>
<output>${project.build.directory}</output>
</configuration>
@@ -294,6 +300,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<goals><goal>cmake-compile</goal></goals>
<configuration>
<source>${basedir}/src</source>
+ <env>
+ <CXXFLAGS>-include cstdint</CXXFLAGS>
+ <PROTOBUF_HOME>${thirdparty.installed}</PROTOBUF_HOME>
+ </env>
<vars>
<CMAKE_C_COMPILER>clang</CMAKE_C_COMPILER>
<CMAKE_CXX_COMPILER>clang++</CMAKE_CXX_COMPILER>
@@ -307,7 +317,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX>
<CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB>
<CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE>
- <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED>
+ <CUSTOM_SNAPPY_PREFIX>${thirdparty.installed}</CUSTOM_SNAPPY_PREFIX>
+ <HDFSPP_LIBRARY_ONLY>ON</HDFSPP_LIBRARY_ONLY>
+ <Boost_USE_STATIC_RUNTIME>ON</Boost_USE_STATIC_RUNTIME>
</vars>
<output>${project.build.directory}/clang</output>
</configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index 9fd2ec0e526..a42ee2e2114 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -34,11 +34,6 @@ enable_testing()
set(CMAKE_CXX_STANDARD 17)
include (CTest)
-set(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/include")
-set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/lib")
-set(HDFSPP_LIBRARY_ONLY 1)
-SET(Boost_USE_STATIC_RUNTIME ON)
-
SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'")
SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 1a0384d50fd..a96290a093b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -66,7 +66,6 @@ check_function_exists(strtoul HAVE_STRTOUL)
# Require snappy.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set(CUSTOM_SNAPPY_PREFIX "${CUSTOM_THIRDPARTY_INSTALLED}")
hadoop_set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
NAMES snappy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 08/09: [fix](tgt)hadoop kerberos support renew TGT by keytab (#173)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit 80e15055b465da628e2d46b80d7d68f837216194
Author: slothever <18...@users.noreply.github.com>
AuthorDate: Fri Jan 12 13:47:31 2024 +0800
[fix](tgt)hadoop kerberos support renew TGT by keytab (#173)
---
.../main/java/org/apache/hadoop/fs/FileSystem.java | 13 +++
.../src/main/native/libhdfs/hdfs.c | 100 ++++++++++++++-------
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 9 ++
3 files changed, 92 insertions(+), 30 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 768fd5b5e1c..1f9e747bc1e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -580,6 +580,19 @@ public abstract class FileSystem extends Configured
});
}
+ public synchronized static FileSystem newInstanceFromKeytab(final URI uri, final Configuration conf,
+ final String principal, final String keytabPath)
+ throws IOException, InterruptedException {
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.loginUserFromKeytab(principal, keytabPath);
+ return UserGroupInformation.getLoginUser().doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override
+ public FileSystem run() throws IOException {
+ return newInstance(uri, conf);
+ }
+ });
+ }
+
/**
* Returns the FileSystem for this URI's scheme and authority.
* The entire URI is passed to the FileSystem instance's initialize method.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index ea00393cfc7..b3ba27f8674 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -36,6 +36,7 @@
#define JMETHOD1(X, R) "(" X ")" R
#define JMETHOD2(X, Y, R) "(" X Y ")" R
#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R
+#define JMETHOD4(X, Y, Z, A, R) "(" X Y Z A")" R
#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path"
@@ -514,6 +515,7 @@ struct hdfsBuilder {
const char *kerbTicketCachePath;
const char *kerb5ConfPath;
const char *keyTabFile;
+ const char *kerbPrincipal;
const char *userName;
struct hdfsBuilderConfOpt *opts;
struct hdfsBuilderConfFileOpt *fileOpts;
@@ -634,6 +636,11 @@ void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName)
bld->userName = userName;
}
+void hdfsBuilderSetPrincipal(struct hdfsBuilder *bld, const char *kerbPrincipal)
+{
+ bld->kerbPrincipal = kerbPrincipal;
+}
+
void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld,
const char *kerbTicketCachePath)
{
@@ -771,7 +778,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
{
JNIEnv *env = 0;
jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL;
- jstring jURIString = NULL, jUserString = NULL, jKeyTabString = NULL;
+ jstring jURIString = NULL, jUserString = NULL, jPrincipalString = NULL, jKeyTabString = NULL;
jvalue jVal;
jthrowable jthr = NULL;
char *cURI = 0, buf[512];
@@ -838,7 +845,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
// fs = FileSytem#getLocal(conf);
jthr = invokeMethod(env, &jVal, STATIC, NULL,
JC_FILE_SYSTEM, "getLocal",
- JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)),
+ JMETHOD1(JPARAM(HADOOP_CONF) , JPARAM(HADOOP_LOCALFS)),
jConfiguration);
if (jthr) {
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -893,23 +900,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
hdfsBuilderToStr(bld, buf, sizeof(buf)));
goto done;
}
- if (bld->kerb5ConfPath && bld->keyTabFile) {
- jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "setConfiguration", JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
- goto done;
- }
- jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
- goto done;
- }
- jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), jUserString, jKeyTabString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
- goto done;
- }
- } else if (bld->kerbTicketCachePath) {
+ if (bld->kerbTicketCachePath) {
jthr = hadoopConfSetStr(env, jConfiguration,
KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath);
if (jthr) {
@@ -922,20 +913,69 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
}
if (bld->forceNewInstance) {
- jthr = invokeMethod(env, &jVal, STATIC, NULL,
- JC_FILE_SYSTEM, "newInstance",
- JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
- JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI,
- jConfiguration, jUserString);
- if (jthr) {
- ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
- "hdfsBuilderConnect(%s)",
- hdfsBuilderToStr(bld, buf, sizeof(buf)));
- goto done;
+ // need kerb5ConfPath to enable kerberos authentication
+ if (bld->kerb5ConfPath && bld->kerbPrincipal && bld->keyTabFile) {
+ jthr = newJavaStr(env, bld->kerbPrincipal, &jPrincipalString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, &jVal, STATIC, NULL,
+ JC_FILE_SYSTEM, "newInstanceFromKeytab",
+ JMETHOD4(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
+ JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI,
+ jConfiguration, jPrincipalString, jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ } else {
+ jthr = invokeMethod(env, &jVal, STATIC, NULL,
+ JC_FILE_SYSTEM, "newInstance",
+ JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF),
+ JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI,
+ jConfiguration, jUserString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
}
jFS = jVal.l;
} else {
- if (bld->keyTabFile && bld->kerb5ConfPath) {
+ if (bld->keyTabFile && bld->kerb5ConfPath && bld->kerbPrincipal) {
+ jthr = newJavaStr(env, bld->kerbPrincipal, &jPrincipalString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
+ "hdfsBuilderConnect(%s)",
+ hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "setConfiguration", JMETHOD1(JPARAM(HADOOP_CONF),JAVA_VOID), jConfiguration);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ jthr = newJavaStr(env, bld->keyTabFile, &jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
+ jthr = invokeMethod(env, NULL, STATIC, NULL, JC_SECURITY_CONFIGURATION, "loginUserFromKeytab", JMETHOD2(JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), jPrincipalString, jKeyTabString);
+ if (jthr) {
+ ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,"hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf)));
+ goto done;
+ }
jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_SYSTEM, "get", JMETHOD1(JPARAM(HADOOP_CONF),
JPARAM(HADOOP_FS)), jConfiguration);
} else {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
index ac2de2d7467..f88545f3ba3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/include/hdfs/hdfs.h
@@ -347,6 +347,15 @@ extern "C" {
*/
LIBHDFS_EXTERNAL
void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName);
+
+ /**
+ * Set the principal to use when connecting to the HDFS cluster.
+ *
+ * @param bld The HDFS builder
+ * @param kerbPrincipal The principal. The string will be shallow-copied.
+ */
+ LIBHDFS_EXTERNAL
+ void hdfsBuilderSetPrincipal(struct hdfsBuilder *bld, const char *kerbPrincipal);
/**
* Set the path to the Kerberos ticket cache to use when connecting to
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris-thirdparty) 02/09: [script] add build script (#71)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.4.0
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
commit 1b4fde83eda6a6d8eb8635f778fd24309b1371e9
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Thu May 18 11:30:17 2023 +0800
[script] add build script (#71)
1. Set `THIRDPARTY_INSTALLED` in `env.h`
Point to the Doris thirdparty installed dir.
2. Run `sh build.sh`
---
build.sh | 60 ++++++++++++++++++++++
env.sh | 20 ++++++++
.../hadoop-hdfs-native-client/pom.xml | 2 +
.../src/main/native/libhdfs/jni_helper.c | 2 +-
.../src/main/native/libhdfspp/CMakeLists.txt | 5 ++
.../src/CMakeLists.txt | 1 +
6 files changed, 89 insertions(+), 1 deletion(-)
diff --git a/build.sh b/build.sh
new file mode 100755
index 00000000000..342f2af5bd0
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to build hadoop
+##############################################################
+
+set -eo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+export SRC_HOME="${ROOT}"
+
+. "${SRC_HOME}/env.sh"
+
+if [[ -z "${THIRDPARTY_INSTALLED}" ]]; then
+ echo "Must set 'THIRDPARTY_INSTALLED' in env.sh"
+ exit -1
+fi
+
+DIST_DIR=${SRC_HOME}/hadoop-dist/target/hadoop-3.3.4/
+LIBHDFS_DIST_DIR=${SRC_HOME}/hadoop-dist/target/hadoop-libhdfs-3.3.4/
+rm -rf ${DIST_DIR}
+rm -rf ${LIBHDFS_DIST_DIR}
+
+export PATH=${THIRDPARTY_INSTALLED}/bin:$PATH
+mvn clean package -Pnative,dist -DskipTests -Dthirdparty.installed=${THIRDPARTY_INSTALLED}/ -Dopenssl.lib=${THIRDPARTY_INSTALLED}/lib/ -e
+
+if [[ ! -d "${DIST_DIR}" ]]; then
+ echo "${DIST_DIR} is missing. Build failed."
+ exit -1
+fi
+
+echo "Finished. Begin to pacakge for libhdfs..."
+mkdir -p ${LIBHDFS_DIST_DIR}/common
+mkdir -p ${LIBHDFS_DIST_DIR}/hdfs
+mkdir -p ${LIBHDFS_DIST_DIR}/include
+cp -r ${DIST_DIR}/share/hadoop/common/* ${LIBHDFS_DIST_DIR}/common/
+cp -r ${DIST_DIR}/share/hadoop/hdfs/* ${LIBHDFS_DIST_DIR}/hdfs/
+cp -r ${DIST_DIR}/include/hdfs.h ${LIBHDFS_DIST_DIR}/include/
+
+echo "Done!"
+echo "The full dist package is under: ${DIST_DIR}"
+echo "The LIBHDFS dist package is under: ${LIBHDFS_DIST_DIR}"
+
diff --git a/env.sh b/env.sh
new file mode 100644
index 00000000000..dd0cd38edc5
--- /dev/null
+++ b/env.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+THIRDPARTY_INSTALLED=
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index cb3838934bf..e9be65d4938 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -237,6 +237,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX>
<CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB>
<CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE>
+ <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED>
</vars>
<output>${project.build.directory}</output>
</configuration>
@@ -306,6 +307,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX>
<CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB>
<CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE>
+ <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED>
</vars>
<output>${project.build.directory}/clang</output>
</configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
index f1f7715e0e1..e754a57664f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c
@@ -238,7 +238,7 @@ jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType,
static jthrowable constructNewObjectOfJclass(JNIEnv *env,
jobject *out, jclass cls, const char *className,
const char *ctorSignature, va_list args) {
- jmethodID mid;
+ jmethodID mid = 0;
jobject jobj;
jthrowable jthr;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index a42ee2e2114..9fd2ec0e526 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -34,6 +34,11 @@ enable_testing()
set(CMAKE_CXX_STANDARD 17)
include (CTest)
+set(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/include")
+set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/lib")
+set(HDFSPP_LIBRARY_ONLY 1)
+SET(Boost_USE_STATIC_RUNTIME ON)
+
SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'")
SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 4c32838afb0..b9a011ef821 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -61,6 +61,7 @@ check_function_exists(strtoul HAVE_STRTOUL)
# Require snappy.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+set(CUSTOM_SNAPPY_PREFIX "${CUSTOM_THIRDPARTY_INSTALLED}")
hadoop_set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
NAMES snappy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org