You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/02 05:19:39 UTC
[doris-thirdparty] branch hadoop-3.3.4 updated: [chore] Build HDFS libraries only (#85)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch hadoop-3.3.4
in repository https://gitbox.apache.org/repos/asf/doris-thirdparty.git
The following commit(s) were added to refs/heads/hadoop-3.3.4 by this push:
new 8b3eea8d [chore] Build HDFS libraries only (#85)
8b3eea8d is described below
commit 8b3eea8d80a374c0b8680f5150d4695901dc36f0
Author: Adonis Ling <ad...@gmail.com>
AuthorDate: Fri Jun 2 13:19:33 2023 +0800
[chore] Build HDFS libraries only (#85)
1. We don't need to build the whole hadoop project. We can build HDFS related libraries to save the build time.
2. Port to GCC-13
---
build.sh | 5 +++--
dev-support/bin/dist-layout-stitching | 18 ++++++++++--------
hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml | 16 ++++++++++++++--
.../src/main/native/libhdfspp/CMakeLists.txt | 5 -----
.../src/CMakeLists.txt | 1 -
5 files changed, 27 insertions(+), 18 deletions(-)
diff --git a/build.sh b/build.sh
index 8d84735c..afff09ef 100755
--- a/build.sh
+++ b/build.sh
@@ -38,9 +38,10 @@ LIBHDFS_DIST_DIR="${SRC_HOME}/hadoop-dist/target/hadoop-libhdfs-3.3.4"
rm -rf "${DIST_DIR}"
rm -rf "${LIBHDFS_DIST_DIR}"
-export PATH=${THIRDPARTY_INSTALLED}/bin:$PATH
-mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip=true \
+mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-common-project -e
+mvn clean package -Pnative,dist -DskipTests -Dmaven.javadoc.skip -f hadoop-hdfs-project \
-Dthirdparty.installed="${THIRDPARTY_INSTALLED}" -Dopenssl.prefix="${THIRDPARTY_INSTALLED}" -e
+mvn clean package -Pdist -DskipTests -Dmaven.javadoc.skip -f hadoop-dist -e
if [[ ! -d "${DIST_DIR}" ]]; then
echo "${DIST_DIR} is missing. Build failed."
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index d4bfd8aa..4d63d18b 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -142,14 +142,16 @@ run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" .
run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* .
run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* .
-# copy client jars as-is
-run mkdir -p "share/hadoop/client"
-run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
-run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
-run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
-
-run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
-run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" .
+if ! ${SKIP_HADOOP_CLIENT_AND_TOOLS:-true}; then
+ # copy client jars as-is
+ run mkdir -p "share/hadoop/client"
+ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
+ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
+ run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
+
+ run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
+ run copy "${ROOT}/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-dist/target/hadoop-dynamometer-dist-${VERSION}" .
+fi
echo
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index ae695a7e..ffa99531 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -210,6 +210,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<goals><goal>cmake-compile</goal></goals>
<configuration>
<source>${basedir}/src</source>
+ <env>
+ <CXXFLAGS>-include cstdint</CXXFLAGS>
+ <PROTOBUF_HOME>${thirdparty.installed}</PROTOBUF_HOME>
+ </env>
<vars>
<GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH>
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
@@ -221,7 +225,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX>
<CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB>
<CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE>
- <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED>
+ <CUSTOM_SNAPPY_PREFIX>${thirdparty.installed}</CUSTOM_SNAPPY_PREFIX>
+ <HDFSPP_LIBRARY_ONLY>ON</HDFSPP_LIBRARY_ONLY>
+ <Boost_USE_STATIC_RUNTIME>ON</Boost_USE_STATIC_RUNTIME>
</vars>
<output>${project.build.directory}</output>
</configuration>
@@ -278,6 +284,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<goals><goal>cmake-compile</goal></goals>
<configuration>
<source>${basedir}/src</source>
+ <env>
+ <CXXFLAGS>-include cstdint</CXXFLAGS>
+ <PROTOBUF_HOME>${thirdparty.installed}</PROTOBUF_HOME>
+ </env>
<vars>
<CMAKE_C_COMPILER>clang</CMAKE_C_COMPILER>
<CMAKE_CXX_COMPILER>clang++</CMAKE_CXX_COMPILER>
@@ -291,7 +301,9 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<CUSTOM_OPENSSL_PREFIX>${openssl.prefix}</CUSTOM_OPENSSL_PREFIX>
<CUSTOM_OPENSSL_LIB>${openssl.lib}</CUSTOM_OPENSSL_LIB>
<CUSTOM_OPENSSL_INCLUDE>${openssl.include}</CUSTOM_OPENSSL_INCLUDE>
- <CUSTOM_THIRDPARTY_INSTALLED>${thirdparty.installed}</CUSTOM_THIRDPARTY_INSTALLED>
+ <CUSTOM_SNAPPY_PREFIX>${thirdparty.installed}</CUSTOM_SNAPPY_PREFIX>
+ <HDFSPP_LIBRARY_ONLY>ON</HDFSPP_LIBRARY_ONLY>
+ <Boost_USE_STATIC_RUNTIME>ON</Boost_USE_STATIC_RUNTIME>
</vars>
<output>${project.build.directory}/clang</output>
</configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index dfb507a4..f64eec10 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -31,11 +31,6 @@ cmake_minimum_required(VERSION 2.8)
enable_testing()
include (CTest)
-set(CMAKE_INCLUDE_PATH ${CMAKE_INCLUDE_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/include")
-set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} "${CUSTOM_THIRDPARTY_INSTALLED}/lib")
-set(HDFSPP_LIBRARY_ONLY 1)
-SET(Boost_USE_STATIC_RUNTIME ON)
-
SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'")
SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 1a0384d5..a96290a0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -66,7 +66,6 @@ check_function_exists(strtoul HAVE_STRTOUL)
# Require snappy.
set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set(CUSTOM_SNAPPY_PREFIX "${CUSTOM_THIRDPARTY_INSTALLED}")
hadoop_set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
NAMES snappy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org