You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bh...@apache.org on 2020/08/11 05:18:53 UTC
[hbase-native-client] 01/02: HBASE-24810: Download Boost with other
dependencies
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-native-client.git
commit 3b81c3d8ce9d817b0da00b4dc1bd0efb1510a296
Author: Bharath Vissapragada <bh...@apache.org>
AuthorDate: Thu Jul 30 21:14:23 2020 -0700
HBASE-24810: Download Boost with other dependencies
Signed-off-by: Josh Elser <el...@apache.org>
---
CMakeLists.txt | 17 ++++++++++---
cmake/BuildTests.cmake | 5 ++--
cmake/DownloadBoost.cmake | 52 +++++++++++++++++++++++++++++++++++++++
cmake/DownloadFolly.cmake | 19 +++++++++++---
cmake/DownloadProtobuf.cmake | 4 ---
cmake/DownloadWangle.cmake | 4 +--
cmake/boost/local/FindBoost.cmake | 36 +++++++++++++++++++++++++++
docker-files/Dockerfile | 2 +-
8 files changed, 122 insertions(+), 17 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e845021..df07059 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,7 +32,6 @@ include(DownloadProject)
include(ExecuteMaven)
include(CheckCXXCompilerFlag)
-
option(DOWNLOAD_DEPENDENCIES "Downloads and builds all dependencies locally " OFF)
option(HBASE_TARGET_TAG "HBase tag to be used if HBASE_HOME is not set" "master")
option(HBASE_HOME "Path to HBase" "")
@@ -56,12 +55,18 @@ endif()
######### Includes
## include the Protobuf generation code
include(ProtobufGen)
+include(DownloadBoost)
include(DownloadFolly)
include(DownloadWangle)
include(DownloadZookeeper)
set(PROJECT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/dependencies")
+set(BOOST_MIN_VERSION "1.6.1")
+# Union of all the libs needed by the current project and dependencies (like folly, wangle etc).
+# Just update this list if more libraries are needed and they will be included and linked automatically.
+set(BOOST_LIBS context coroutine thread system filesystem regex program_options)
+
if (DOWNLOAD_DEPENDENCIES)
## we want to find the system protoc
download_project(PROJ Protobuf PREFIX "${PROJECT_PREFIX}" IS_AUTOGEN GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git" GIT_TAG "3.5.1.1")
@@ -78,11 +83,11 @@ if (DOWNLOAD_DEPENDENCIES)
set(PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_BINARY_DIR}/bin/protoc" CACHE STRING "" FORCE)
## Add CMAKE_MODULE_PATHS
+ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/wangle/local")
-
else()
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/system")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/system")
@@ -154,16 +159,16 @@ endif (OPENSSL_FOUND)
if (DOWNLOAD_DEPENDENCIES)
+ download_boost(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} "${BOOST_LIBS}")
download_folly(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
download_wangle(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
download_zookeeper(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
endif(DOWNLOAD_DEPENDENCIES)
-set(BOOST_MIN_VERSION "1.6.1")
# ensure we have required dependencies
find_package(Threads)
-find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS context thread system filesystem regex)
+find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS "${BOOST_LIBS}")
find_package(LibEvent REQUIRED)
find_package(Gflags REQUIRED)
if (DOWNLOAD_DEPENDENCIES)
@@ -309,6 +314,9 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/bin/cpplint.sh)
if (DOWNLOAD_DEPENDENCIES)
+ add_dependencies(facebook-folly-proj boost)
+ add_dependencies(hbaseclient-static boost)
+ add_dependencies(hbaseclient-shared boost)
add_dependencies(hbaseclient-static Protobuf)
add_dependencies(hbaseclient-shared Protobuf)
add_dependencies(facebook-wangle-proj facebook-folly-proj)
@@ -341,3 +349,4 @@ install(
DIRECTORY "${CMAKE_BINARY_DIR_GEN}"
DESTINATION include/hbase/if
FILES_MATCHING PATTERN "hbase/if/*.h")
+
diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake
index e7b4243..4808fc3 100644
--- a/cmake/BuildTests.cmake
+++ b/cmake/BuildTests.cmake
@@ -53,7 +53,7 @@ function(createTests testName)
#target_link_libraries(${testName} ${PROTOBUF_LIBRARY})
#${PROTOBUF_LIBRARY}
- target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}
+ target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_DL_LIBS}
${Java_LIBRARIES}
${JNI_LIBRARIES}
${Boost_LIBRARIES}
@@ -63,7 +63,8 @@ function(createTests testName)
${GFLAGS_SHARED_LIB}
${KRB5_LIBRARIES}
${ZOOKEEPER_LIBRARIES} ${OPENSSL_LIBRARIES}
- ${GLOG_SHARED_LIB})
+ ${GLOG_SHARED_LIB}
+ ${CMAKE_THREAD_LIBS_INIT})
endfunction()
enable_testing(test)
SET(TEST_DIR ${CMAKE_SOURCE_DIR}/src/test)
diff --git a/cmake/DownloadBoost.cmake b/cmake/DownloadBoost.cmake
new file mode 100644
index 0000000..388a511
--- /dev/null
+++ b/cmake/DownloadBoost.cmake
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+## Download Boost.
+## SOURCE_DIR is typically the cmake source directory
+## BINARY_DIR is the build directory, typically 'build'
+## Sets BOOST_ROOT, search prefix for FindBoost.
+
+function(download_boost SOURCE_DIR BUILD_DIR BOOST_LIBS)
+ set(BOOST_DOWNLOAD_DIR "${BUILD_DIR}/dependencies/boost-download")
+ set(BOOST_SOURCE_DIR "${BUILD_DIR}/dependencies/boost-src")
+ set(BOOST_INSTALL_DIR "${BUILD_DIR}/dependencies/boost-install")
+
+ set(CFLAGS "-fPIC")
+ set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+ # Only compile and install the needed libs.
+ set(LIBS_TO_COMPILE "")
+ foreach(lib ${BOOST_LIBS})
+ string(APPEND LIBS_TO_COMPILE --with-${lib} " ")
+ endforeach()
+
+ separate_arguments(BUILD_CMD UNIX_COMMAND
+ "./b2 cflags='${CFLAGS}' cxxflags='${CXXFLAGS}' variant=release link=static threading=multi ${LIBS_TO_COMPILE} install")
+
+ ExternalProject_Add(boost
+ URL "https://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz"
+ PREFIX "${BUILD_DIR}/dependencies"
+ DOWNLOAD_DIR ${BOOST_DOWNLOAD_DIR}
+ BUILD_IN_SOURCE true
+ SOURCE_DIR ${BOOST_SOURCE_DIR}
+ INSTALL_DIR ${BOOST_INSTALL_DIR}
+ CONFIGURE_COMMAND ./bootstrap.sh --prefix=${BOOST_INSTALL_DIR}
+ BUILD_COMMAND ${BUILD_CMD}
+ INSTALL_COMMAND ""
+ )
+ set(BOOST_ROOT ${BOOST_INSTALL_DIR} PARENT_SCOPE)
+endfunction(download_boost)
diff --git a/cmake/DownloadFolly.cmake b/cmake/DownloadFolly.cmake
index 16f479b..1ff5556 100644
--- a/cmake/DownloadFolly.cmake
+++ b/cmake/DownloadFolly.cmake
@@ -20,6 +20,20 @@
## BUILD_DIR is the build directory, typically 'build'
function(download_folly SOURCE_DIR BUILD_DIR)
+
+ if (DOWNLOAD_DEPENDENCIES)
+ # Add custom boost include and lib paths.
+ set(CFLAGS "-fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(LDFLAGS "-L${BOOST_ROOT}/lib")
+ set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install
+ --with-boost-libdir=${BOOST_ROOT}/lib CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS} LDFLAGS=${LDFLAGS})
+ else()
+ set(CFLAGS "-fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS})
+ endif()
+
ExternalProject_Add(
facebook-folly-proj
# TODO: Source version information from cmake file.
@@ -27,10 +41,7 @@ function(download_folly SOURCE_DIR BUILD_DIR)
PREFIX "${BUILD_DIR}/dependencies"
SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-src"
BINARY_DIR ${BUILD_DIR}/dependencies/facebook-folly-proj-src/folly
- CONFIGURE_COMMAND autoreconf -ivf
- COMMAND ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install
- "CFLAGS=-fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually
- "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually
+ CONFIGURE_COMMAND autoreconf -ivf COMMAND ${CONFIGURE_CMD}
UPDATE_COMMAND ""
)
set(FOLLY_ROOT_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-install" CACHE STRING "" FORCE)
diff --git a/cmake/DownloadProtobuf.cmake b/cmake/DownloadProtobuf.cmake
index ca8260c..09e7c67 100644
--- a/cmake/DownloadProtobuf.cmake
+++ b/cmake/DownloadProtobuf.cmake
@@ -22,8 +22,6 @@
#################### PROTOBUF
function(download_protobuf SOURCE_DIR BINARY_DIR)
-
-
ExternalProject_Add(
Protobuf
GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git"
@@ -46,7 +44,5 @@ function(download_protobuf SOURCE_DIR BINARY_DIR)
set(PROTOBUF_INCLUDE_DIRS "${CMAKE_CURRENT_BINARY_DIR}/dependencies/protobuf/include" CACHE STRING "" FORCE)
add_dependencies(protobuf Protobuf)
set(PROTOBUF_FOUND TRUE CACHE STRING "" FORCE)
-
-
endfunction(download_protobuf)
diff --git a/cmake/DownloadWangle.cmake b/cmake/DownloadWangle.cmake
index 1bfff5f..74c97a6 100644
--- a/cmake/DownloadWangle.cmake
+++ b/cmake/DownloadWangle.cmake
@@ -24,7 +24,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR)
set(WANGLE_SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-src")
set(WANGLE_INSTALL_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-install")
if (DOWNLOAD_DEPENDENCIES)
- set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake")
+ set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local/FindBoost.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake")
else()
set(PATCH_FOLLY "")
endif()
@@ -37,7 +37,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR)
SOURCE_DIR ${WANGLE_SOURCE_DIR}
PATCH_COMMAND ${PATCH_FOLLY}
INSTALL_DIR ${WANGLE_INSTALL_DIR}
- CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source.
+ CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DBOOST_ROOT=${BOOST_ROOT} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source.
)
set(WANGLE_ROOT_DIR ${WANGLE_INSTALL_DIR} CACHE STRING "" FORCE)
endfunction(download_wangle)
diff --git a/cmake/boost/local/FindBoost.cmake b/cmake/boost/local/FindBoost.cmake
new file mode 100644
index 0000000..60d7ee0
--- /dev/null
+++ b/cmake/boost/local/FindBoost.cmake
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Stubs to allow us to find Boost libs
+
+set(Boost_INCLUDE_DIRS "${BOOST_ROOT}/include" CACHE STRING "" FORCE)
+set(Boost_INCLUDE_DIR "${BOOST_ROOT}/include" CACHE STRING "" FORCE)
+
+set(Boost_LIBRARIES "" CACHE STRING "" FORCE)
+foreach(COMPONENT ${Boost_FIND_COMPONENTS})
+ list(APPEND Boost_LIBRARIES "${BOOST_ROOT}/lib/${BYPRODUCT_PREFIX}boost_${COMPONENT}${BYPRODUCT_SUFFIX}")
+endforeach()
+
+set(Boost_FOUND "true" CACHE STRING "" FORCE)
+
+mark_as_advanced(
+ Boost_FOUND
+ Boost_INCLUDE_DIR
+ Boost_INCLUDE_DIRS
+ Boost_LIBRARIES
+)
+message("-- Boost found, ${Boost_LIBRARIES}")
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index 3cad6d2..82545d7 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -30,7 +30,7 @@ ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
RUN apt-get update && \
apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format valgrind \
wget libgflags-dev libgoogle-glog-dev dh-autoreconf pkg-config libssl-dev build-essential \
- libboost-all-dev libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \
+ libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \
pip install yapf && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \