You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by se...@apache.org on 2021/05/14 21:45:02 UTC
[bigtop] branch master updated: BIGTOP-3457. Add libhdfspp packages
to hadoop. (#770)
This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 163c4bf BIGTOP-3457. Add libhdfspp packages to hadoop. (#770)
163c4bf is described below
commit 163c4bf7d2915ced590dbc430c1f872bfe05025c
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Sat May 15 06:44:53 2021 +0900
BIGTOP-3457. Add libhdfspp packages to hadoop. (#770)
* BIGTOP-3457. Add libhdfspp packages to hadoop.
* addressed missing dependency information error.
* revert HADOOP-17196 to fix build failure on CentOS 7.
---
.../src/common/hadoop/do-component-build | 2 +
.../src/common/hadoop/install_hadoop.sh | 4 +-
.../common/hadoop/patch9-revert-HADOOP-17196.diff | 73 ++++++++++++++++++++++
bigtop-packages/src/deb/hadoop/control | 12 ++++
bigtop-packages/src/deb/hadoop/libhdfs0.install | 2 +-
bigtop-packages/src/deb/hadoop/libhdfspp-dev.dirs | 1 +
.../src/deb/hadoop/libhdfspp-dev.install | 1 +
bigtop-packages/src/deb/hadoop/libhdfspp.dirs | 1 +
bigtop-packages/src/deb/hadoop/libhdfspp.install | 1 +
bigtop-packages/src/deb/hadoop/rules | 3 +
bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec | 29 ++++++++-
11 files changed, 124 insertions(+), 5 deletions(-)
diff --git a/bigtop-packages/src/common/hadoop/do-component-build b/bigtop-packages/src/common/hadoop/do-component-build
index 221ea7f..3f188e2 100644
--- a/bigtop-packages/src/common/hadoop/do-component-build
+++ b/bigtop-packages/src/common/hadoop/do-component-build
@@ -140,3 +140,5 @@ cp -r target/staging/hadoop-project build/share/doc
# Copy fuse output to the build directory
cp hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/fuse-dfs/fuse_dfs build/bin
+
+cp -r hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp build/include/
diff --git a/bigtop-packages/src/common/hadoop/install_hadoop.sh b/bigtop-packages/src/common/hadoop/install_hadoop.sh
index a85f38d..68152a4 100755
--- a/bigtop-packages/src/common/hadoop/install_hadoop.sh
+++ b/bigtop-packages/src/common/hadoop/install_hadoop.sh
@@ -238,7 +238,8 @@ cp -a ${BUILD_DIR}/sbin/mr-jobhistory-daemon.sh ${MAPREDUCE_DIR}/sbin
# native libs
install -d -m 0755 ${SYSTEM_LIB_DIR}
install -d -m 0755 ${HADOOP_NATIVE_LIB_DIR}
-for library in libhdfs.so.0.0.0; do
+
+for library in libhdfs.so.0.0.0 libhdfspp.so.0.1.0 ; do
cp ${BUILD_DIR}/lib/native/${library} ${SYSTEM_LIB_DIR}/
ldconfig -vlN ${SYSTEM_LIB_DIR}/${library}
ln -s ${library} ${SYSTEM_LIB_DIR}/${library/.so.*/}.so
@@ -246,6 +247,7 @@ done
install -d -m 0755 ${SYSTEM_INCLUDE_DIR}
cp ${BUILD_DIR}/include/hdfs.h ${SYSTEM_INCLUDE_DIR}/
+cp -r ${BUILD_DIR}/include/hdfspp ${SYSTEM_INCLUDE_DIR}/
cp ${BUILD_DIR}/lib/native/*.a ${HADOOP_NATIVE_LIB_DIR}/
for library in `cd ${BUILD_DIR}/lib/native ; ls libsnappy.so.1.* 2>/dev/null` libhadoop.so.1.0.0 libnativetask.so.1.0.0; do
diff --git a/bigtop-packages/src/common/hadoop/patch9-revert-HADOOP-17196.diff b/bigtop-packages/src/common/hadoop/patch9-revert-HADOOP-17196.diff
new file mode 100644
index 0000000..933bc7b
--- /dev/null
+++ b/bigtop-packages/src/common/hadoop/patch9-revert-HADOOP-17196.diff
@@ -0,0 +1,73 @@
+commit f790a72a4c68e242b5d54323c607e18d5dbda394
+Author: Masatake Iwasaki <iw...@apache.org>
+Date: Fri May 14 11:48:47 2021 +0000
+
+ Revert "HADOOP-17196. Fix C/C++ standard warnings (#2208)"
+
+ This reverts commit ff907b310d905fde648dc6db8954e425c24e3bf3.
+
+diff --git a/hadoop-common-project/hadoop-common/HadoopCommon.cmake b/hadoop-common-project/hadoop-common/HadoopCommon.cmake
+index 7628ecf..4de70ac 100644
+--- a/hadoop-common-project/hadoop-common/HadoopCommon.cmake
++++ b/hadoop-common-project/hadoop-common/HadoopCommon.cmake
+@@ -193,7 +193,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+ # Solaris flags. 64-bit compilation is mandatory, and is checked earlier.
+ hadoop_add_compiler_flags("-m64 -D_POSIX_C_SOURCE=200112L -D__EXTENSIONS__ -D_POSIX_PTHREAD_SEMANTICS")
+- set(CMAKE_CXX_STANDARD 98)
++ set(CMAKE_CXX_FLAGS "-std=gnu++98 ${CMAKE_CXX_FLAGS}")
+ hadoop_add_linker_flags("-m64")
+
+ # CMAKE_SYSTEM_PROCESSOR is set to the output of 'uname -p', which on Solaris is
+@@ -212,4 +212,4 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "SunOS")
+ endif()
+
+ # Set GNU99 as the C standard to use
+-set(CMAKE_C_STANDARD 99)
+\ No newline at end of file
++set(CMAKE_C_FLAGS "-std=gnu99 ${CMAKE_C_FLAGS}")
+\ No newline at end of file
+diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+index 18396c7..1676e31 100644
+--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+@@ -152,8 +152,7 @@ add_subdirectory(main/native/libhdfs-tests)
+ # Temporary fix to disable Libhdfs++ build on older systems that do not support thread_local
+ include(CheckCXXSourceCompiles)
+ unset (THREAD_LOCAL_SUPPORTED CACHE)
+-set (CMAKE_CXX_STANDARD 11)
+-set (CMAKE_CXX_STANDARD_REQUIRED ON)
++set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11")
+ set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+ check_cxx_source_compiles(
+ "#include <thread>
+diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+index 2da5b6bb..c17f9d3 100644
+--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+@@ -51,8 +51,7 @@ include(CheckCXXSourceCompiles)
+
+ # Check if thread_local is supported
+ unset (THREAD_LOCAL_SUPPORTED CACHE)
+-set (CMAKE_CXX_STANDARD 11)
+-set (CMAKE_CXX_STANDARD_REQUIRED ON)
++set (CMAKE_REQUIRED_DEFINITIONS "-std=c++11")
+ set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT})
+ check_cxx_source_compiles(
+ "#include <thread>
+@@ -148,13 +147,12 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0")
+ set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0")
+
+ if(UNIX)
+-set (CMAKE_CXX_STANDARD 11)
+-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -g -fPIC -fno-strict-aliasing")
++set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -std=c++11 -g -fPIC -fno-strict-aliasing")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing")
+ endif()
+
+ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+- set(CMAKE_CXX_STANDARD 11)
++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+ add_definitions(-DASIO_HAS_STD_ADDRESSOF -DASIO_HAS_STD_ARRAY -DASIO_HAS_STD_ATOMIC -DASIO_HAS_CSTDINT -DASIO_HAS_STD_SHARED_PTR -DASIO_HAS_STD_TYPE_TRAITS -DASIO_HAS_VARIADIC_TEMPLATES -DASIO_HAS_STD_FUNCTION -DASIO_HAS_STD_CHRONO -DASIO_HAS_STD_SYSTEM_ERROR)
+ endif ()
+
diff --git a/bigtop-packages/src/deb/hadoop/control b/bigtop-packages/src/deb/hadoop/control
index 8a35d96..7b2a538 100644
--- a/bigtop-packages/src/deb/hadoop/control
+++ b/bigtop-packages/src/deb/hadoop/control
@@ -205,6 +205,18 @@ Depends: hadoop (= ${binary:Version}), libhdfs0 (= ${binary:Version})
Description: Development support for libhdfs0
Includes examples and header files for accessing HDFS from C
+Package: libhdfspp
+Architecture: any
+Description: Hadoop Filesystem Library for C++
+ Hadoop Filesystem Library for C++
+
+Package: libhdfspp-dev
+Architecture: any
+Section: libdevel
+Depends: libhdfspp (= ${binary:Version})
+Description: Development support for libhdfspp
+ Includes header files for accessing HDFS from C++
+
Package: hadoop-httpfs
Architecture: any
Depends: hadoop-hdfs (= ${binary:Version})
diff --git a/bigtop-packages/src/deb/hadoop/libhdfs0.install b/bigtop-packages/src/deb/hadoop/libhdfs0.install
index 0e848d6..4abd42e 100644
--- a/bigtop-packages/src/deb/hadoop/libhdfs0.install
+++ b/bigtop-packages/src/deb/hadoop/libhdfs0.install
@@ -1 +1 @@
-/usr/lib/libhdfs.so*
+/usr/lib/libhdfs.*
diff --git a/bigtop-packages/src/deb/hadoop/libhdfspp-dev.dirs b/bigtop-packages/src/deb/hadoop/libhdfspp-dev.dirs
new file mode 100644
index 0000000..dfc9182
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/libhdfspp-dev.dirs
@@ -0,0 +1 @@
+/usr/include/hdfspp
diff --git a/bigtop-packages/src/deb/hadoop/libhdfspp-dev.install b/bigtop-packages/src/deb/hadoop/libhdfspp-dev.install
new file mode 100644
index 0000000..dfc9182
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/libhdfspp-dev.install
@@ -0,0 +1 @@
+/usr/include/hdfspp
diff --git a/bigtop-packages/src/deb/hadoop/libhdfspp.dirs b/bigtop-packages/src/deb/hadoop/libhdfspp.dirs
new file mode 100644
index 0000000..fce258f
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/libhdfspp.dirs
@@ -0,0 +1 @@
+/usr/lib/
diff --git a/bigtop-packages/src/deb/hadoop/libhdfspp.install b/bigtop-packages/src/deb/hadoop/libhdfspp.install
new file mode 100644
index 0000000..8a938d9
--- /dev/null
+++ b/bigtop-packages/src/deb/hadoop/libhdfspp.install
@@ -0,0 +1 @@
+/usr/lib/libhdfspp.*
diff --git a/bigtop-packages/src/deb/hadoop/rules b/bigtop-packages/src/deb/hadoop/rules
index 6f6d4c0..fe0f301 100755
--- a/bigtop-packages/src/deb/hadoop/rules
+++ b/bigtop-packages/src/deb/hadoop/rules
@@ -87,3 +87,6 @@ override_dh_install: $(hadoop_svcs)
override_dh_strip:
dh_strip --no-automatic-dbgsym
+
+override_dh_shlibdeps:
+ dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info
diff --git a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
index 6f59500..877c3e0 100644
--- a/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
+++ b/bigtop-packages/src/rpm/hadoop/SPECS/hadoop.spec
@@ -478,6 +478,24 @@ Requires: hadoop = %{version}-%{release}, hadoop-libhdfs = %{version}-%{release}
%description libhdfs-devel
Includes examples and header files for accessing HDFS from C
+%package libhdfspp
+Summary: Hadoop Filesystem Library for C++
+Group: Development/Libraries
+
+%description libhdfs-devel
+Includes examples and header files for accessing HDFS from C
+
+%package libhdfspp-devel
+Summary: Development support for libhdfspp
+Group: Development/Libraries
+Requires: %{name}-libhdfspp = %{version}-%{release}
+
+%description libhdfspp
+Hadoop Filesystem Library for C++
+
+%description libhdfspp-devel
+Includes header files for accessing HDFS from C++
+
%package hdfs-fuse
Summary: Mountable HDFS
Group: Development/Libraries
@@ -812,16 +830,21 @@ fi
%files libhdfs
%defattr(-,root,root)
-%{_libdir}/libhdfs*
+%{_libdir}/libhdfs.*
%files libhdfs-devel
%{_includedir}/hdfs.h
#%doc %{_docdir}/libhdfs-%{hadoop_version}
+%files libhdfspp
+%defattr(-,root,root)
+%{_libdir}/libhdfspp.*
+
+%files libhdfspp-devel
+%{_includedir}/hdfspp
+
%files hdfs-fuse
%defattr(-,root,root)
%attr(0644,root,root) %config(noreplace) /etc/default/hadoop-fuse
%attr(0755,root,root) %{lib_hadoop}/bin/fuse_dfs
%attr(0755,root,root) %{bin_hadoop}/hadoop-fuse-dfs
-
-