You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bh...@apache.org on 2020/08/11 05:18:52 UTC
[hbase-native-client] branch master updated (3b86174 -> 926a6f9)
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-native-client.git.
from 3b86174 HBASE-24642 (addendum): Do not run the container as root (#13)
new 3b81c3d HBASE-24810: Download Boost with other dependencies
new 926a6f9 HBASE-24642 (addendum): Reduce make concurrency and increase container mem limit
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 17 +++++--
bin/hbase-native-client-personality.sh | 9 +++-
cmake/BuildTests.cmake | 5 ++-
cmake/DownloadBoost.cmake | 52 ++++++++++++++++++++++
cmake/DownloadFolly.cmake | 19 ++++++--
cmake/DownloadProtobuf.cmake | 4 --
cmake/DownloadWangle.cmake | 4 +-
.../local/FindBoost.cmake} | 30 +++++++------
docker-files/Dockerfile | 2 +-
9 files changed, 110 insertions(+), 32 deletions(-)
create mode 100644 cmake/DownloadBoost.cmake
copy cmake/{wangle/local/FindWangle.cmake => boost/local/FindBoost.cmake} (58%)
[hbase-native-client] 01/02: HBASE-24810: Download Boost with other
dependencies
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-native-client.git
commit 3b81c3d8ce9d817b0da00b4dc1bd0efb1510a296
Author: Bharath Vissapragada <bh...@apache.org>
AuthorDate: Thu Jul 30 21:14:23 2020 -0700
HBASE-24810: Download Boost with other dependencies
Signed-off-by: Josh Elser <el...@apache.org>
---
CMakeLists.txt | 17 ++++++++++---
cmake/BuildTests.cmake | 5 ++--
cmake/DownloadBoost.cmake | 52 +++++++++++++++++++++++++++++++++++++++
cmake/DownloadFolly.cmake | 19 +++++++++++---
cmake/DownloadProtobuf.cmake | 4 ---
cmake/DownloadWangle.cmake | 4 +--
cmake/boost/local/FindBoost.cmake | 36 +++++++++++++++++++++++++++
docker-files/Dockerfile | 2 +-
8 files changed, 122 insertions(+), 17 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e845021..df07059 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -32,7 +32,6 @@ include(DownloadProject)
include(ExecuteMaven)
include(CheckCXXCompilerFlag)
-
option(DOWNLOAD_DEPENDENCIES "Downloads and builds all dependencies locally " OFF)
option(HBASE_TARGET_TAG "HBase tag to be used if HBASE_HOME is not set" "master")
option(HBASE_HOME "Path to HBase" "")
@@ -56,12 +55,18 @@ endif()
######### Includes
## include the Protobuf generation code
include(ProtobufGen)
+include(DownloadBoost)
include(DownloadFolly)
include(DownloadWangle)
include(DownloadZookeeper)
set(PROJECT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/dependencies")
+set(BOOST_MIN_VERSION "1.6.1")
+# Union of all the libs needed by the current project and dependencies (like folly, wangle etc).
+# Just update this list if more libraries are needed and they will be included and linked automatically.
+set(BOOST_LIBS context coroutine thread system filesystem regex program_options)
+
if (DOWNLOAD_DEPENDENCIES)
## we want to find the system protoc
download_project(PROJ Protobuf PREFIX "${PROJECT_PREFIX}" IS_AUTOGEN GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git" GIT_TAG "3.5.1.1")
@@ -78,11 +83,11 @@ if (DOWNLOAD_DEPENDENCIES)
set(PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_BINARY_DIR}/bin/protoc" CACHE STRING "" FORCE)
## Add CMAKE_MODULE_PATHS
+ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/wangle/local")
-
else()
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/system")
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/system")
@@ -154,16 +159,16 @@ endif (OPENSSL_FOUND)
if (DOWNLOAD_DEPENDENCIES)
+ download_boost(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} "${BOOST_LIBS}")
download_folly(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
download_wangle(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
download_zookeeper(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
endif(DOWNLOAD_DEPENDENCIES)
-set(BOOST_MIN_VERSION "1.6.1")
# ensure we have required dependencies
find_package(Threads)
-find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS context thread system filesystem regex)
+find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS "${BOOST_LIBS}")
find_package(LibEvent REQUIRED)
find_package(Gflags REQUIRED)
if (DOWNLOAD_DEPENDENCIES)
@@ -309,6 +314,9 @@ add_custom_target(
COMMAND ${CMAKE_SOURCE_DIR}/bin/cpplint.sh)
if (DOWNLOAD_DEPENDENCIES)
+ add_dependencies(facebook-folly-proj boost)
+ add_dependencies(hbaseclient-static boost)
+ add_dependencies(hbaseclient-shared boost)
add_dependencies(hbaseclient-static Protobuf)
add_dependencies(hbaseclient-shared Protobuf)
add_dependencies(facebook-wangle-proj facebook-folly-proj)
@@ -341,3 +349,4 @@ install(
DIRECTORY "${CMAKE_BINARY_DIR_GEN}"
DESTINATION include/hbase/if
FILES_MATCHING PATTERN "hbase/if/*.h")
+
diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake
index e7b4243..4808fc3 100644
--- a/cmake/BuildTests.cmake
+++ b/cmake/BuildTests.cmake
@@ -53,7 +53,7 @@ function(createTests testName)
#target_link_libraries(${testName} ${PROTOBUF_LIBRARY})
#${PROTOBUF_LIBRARY}
- target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS}
+ target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_DL_LIBS}
${Java_LIBRARIES}
${JNI_LIBRARIES}
${Boost_LIBRARIES}
@@ -63,7 +63,8 @@ function(createTests testName)
${GFLAGS_SHARED_LIB}
${KRB5_LIBRARIES}
${ZOOKEEPER_LIBRARIES} ${OPENSSL_LIBRARIES}
- ${GLOG_SHARED_LIB})
+ ${GLOG_SHARED_LIB}
+ ${CMAKE_THREAD_LIBS_INIT})
endfunction()
enable_testing(test)
SET(TEST_DIR ${CMAKE_SOURCE_DIR}/src/test)
diff --git a/cmake/DownloadBoost.cmake b/cmake/DownloadBoost.cmake
new file mode 100644
index 0000000..388a511
--- /dev/null
+++ b/cmake/DownloadBoost.cmake
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+## Download Boost.
+## SOURCE_DIR is typically the cmake source directory
+## BINARY_DIR is the build directory, typically 'build'
+## Sets BOOST_ROOT, search prefix for FindBoost.
+
+function(download_boost SOURCE_DIR BUILD_DIR BOOST_LIBS)
+ set(BOOST_DOWNLOAD_DIR "${BUILD_DIR}/dependencies/boost-download")
+ set(BOOST_SOURCE_DIR "${BUILD_DIR}/dependencies/boost-src")
+ set(BOOST_INSTALL_DIR "${BUILD_DIR}/dependencies/boost-install")
+
+ set(CFLAGS "-fPIC")
+ set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+ # Only compile and install the needed libs.
+ set(LIBS_TO_COMPILE "")
+ foreach(lib ${BOOST_LIBS})
+ string(APPEND LIBS_TO_COMPILE --with-${lib} " ")
+ endforeach()
+
+ separate_arguments(BUILD_CMD UNIX_COMMAND
+ "./b2 cflags='${CFLAGS}' cxxflags='${CXXFLAGS}' variant=release link=static threading=multi ${LIBS_TO_COMPILE} install")
+
+ ExternalProject_Add(boost
+ URL "https://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz"
+ PREFIX "${BUILD_DIR}/dependencies"
+ DOWNLOAD_DIR ${BOOST_DOWNLOAD_DIR}
+ BUILD_IN_SOURCE true
+ SOURCE_DIR ${BOOST_SOURCE_DIR}
+ INSTALL_DIR ${BOOST_INSTALL_DIR}
+ CONFIGURE_COMMAND ./bootstrap.sh --prefix=${BOOST_INSTALL_DIR}
+ BUILD_COMMAND ${BUILD_CMD}
+ INSTALL_COMMAND ""
+ )
+ set(BOOST_ROOT ${BOOST_INSTALL_DIR} PARENT_SCOPE)
+endfunction(download_boost)
diff --git a/cmake/DownloadFolly.cmake b/cmake/DownloadFolly.cmake
index 16f479b..1ff5556 100644
--- a/cmake/DownloadFolly.cmake
+++ b/cmake/DownloadFolly.cmake
@@ -20,6 +20,20 @@
## BUILD_DIR is the build directory, typically 'build'
function(download_folly SOURCE_DIR BUILD_DIR)
+
+ if (DOWNLOAD_DEPENDENCIES)
+ # Add custom boost include and lib paths.
+ set(CFLAGS "-fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(LDFLAGS "-L${BOOST_ROOT}/lib")
+ set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install
+ --with-boost-libdir=${BOOST_ROOT}/lib CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS} LDFLAGS=${LDFLAGS})
+ else()
+ set(CFLAGS "-fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+ set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS})
+ endif()
+
ExternalProject_Add(
facebook-folly-proj
# TODO: Source version information from cmake file.
@@ -27,10 +41,7 @@ function(download_folly SOURCE_DIR BUILD_DIR)
PREFIX "${BUILD_DIR}/dependencies"
SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-src"
BINARY_DIR ${BUILD_DIR}/dependencies/facebook-folly-proj-src/folly
- CONFIGURE_COMMAND autoreconf -ivf
- COMMAND ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install
- "CFLAGS=-fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually
- "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually
+ CONFIGURE_COMMAND autoreconf -ivf COMMAND ${CONFIGURE_CMD}
UPDATE_COMMAND ""
)
set(FOLLY_ROOT_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-install" CACHE STRING "" FORCE)
diff --git a/cmake/DownloadProtobuf.cmake b/cmake/DownloadProtobuf.cmake
index ca8260c..09e7c67 100644
--- a/cmake/DownloadProtobuf.cmake
+++ b/cmake/DownloadProtobuf.cmake
@@ -22,8 +22,6 @@
#################### PROTOBUF
function(download_protobuf SOURCE_DIR BINARY_DIR)
-
-
ExternalProject_Add(
Protobuf
GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git"
@@ -46,7 +44,5 @@ function(download_protobuf SOURCE_DIR BINARY_DIR)
set(PROTOBUF_INCLUDE_DIRS "${CMAKE_CURRENT_BINARY_DIR}/dependencies/protobuf/include" CACHE STRING "" FORCE)
add_dependencies(protobuf Protobuf)
set(PROTOBUF_FOUND TRUE CACHE STRING "" FORCE)
-
-
endfunction(download_protobuf)
diff --git a/cmake/DownloadWangle.cmake b/cmake/DownloadWangle.cmake
index 1bfff5f..74c97a6 100644
--- a/cmake/DownloadWangle.cmake
+++ b/cmake/DownloadWangle.cmake
@@ -24,7 +24,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR)
set(WANGLE_SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-src")
set(WANGLE_INSTALL_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-install")
if (DOWNLOAD_DEPENDENCIES)
- set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake")
+ set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local/FindBoost.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake")
else()
set(PATCH_FOLLY "")
endif()
@@ -37,7 +37,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR)
SOURCE_DIR ${WANGLE_SOURCE_DIR}
PATCH_COMMAND ${PATCH_FOLLY}
INSTALL_DIR ${WANGLE_INSTALL_DIR}
- CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source.
+ CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DBOOST_ROOT=${BOOST_ROOT} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source.
)
set(WANGLE_ROOT_DIR ${WANGLE_INSTALL_DIR} CACHE STRING "" FORCE)
endfunction(download_wangle)
diff --git a/cmake/boost/local/FindBoost.cmake b/cmake/boost/local/FindBoost.cmake
new file mode 100644
index 0000000..60d7ee0
--- /dev/null
+++ b/cmake/boost/local/FindBoost.cmake
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Stubs to allow us to find Boost libs
+
+set(Boost_INCLUDE_DIRS "${BOOST_ROOT}/include" CACHE STRING "" FORCE)
+set(Boost_INCLUDE_DIR "${BOOST_ROOT}/include" CACHE STRING "" FORCE)
+
+set(Boost_LIBRARIES "" CACHE STRING "" FORCE)
+foreach(COMPONENT ${Boost_FIND_COMPONENTS})
+ list(APPEND Boost_LIBRARIES "${BOOST_ROOT}/lib/${BYPRODUCT_PREFIX}boost_${COMPONENT}${BYPRODUCT_SUFFIX}")
+endforeach()
+
+set(Boost_FOUND "true" CACHE STRING "" FORCE)
+
+mark_as_advanced(
+ Boost_FOUND
+ Boost_INCLUDE_DIR
+ Boost_INCLUDE_DIRS
+ Boost_LIBRARIES
+)
+message("-- Boost found, ${Boost_LIBRARIES}")
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index 3cad6d2..82545d7 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -30,7 +30,7 @@ ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
RUN apt-get update && \
apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format valgrind \
wget libgflags-dev libgoogle-glog-dev dh-autoreconf pkg-config libssl-dev build-essential \
- libboost-all-dev libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \
+ libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \
pip install yapf && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
[hbase-native-client] 02/02: HBASE-24642 (addendum): Reduce make
concurrency and increase container mem limit
Posted by bh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase-native-client.git
commit 926a6f9dc66eeb551769b1ab15dc1a86352df175
Author: Bharath Vissapragada <bh...@apache.org>
AuthorDate: Sun Aug 2 13:42:58 2020 -0700
HBASE-24642 (addendum): Reduce make concurrency and increase container mem limit
Signed-off-by: Josh Elser <el...@apache.org>
---
bin/hbase-native-client-personality.sh | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/bin/hbase-native-client-personality.sh b/bin/hbase-native-client-personality.sh
index af6ee3f..b3bcf33 100755
--- a/bin/hbase-native-client-personality.sh
+++ b/bin/hbase-native-client-personality.sh
@@ -73,7 +73,9 @@ function personality_globals
# Passed to cmake command using a custom personality.
CMAKE_ARGS="-DDOWNLOAD_DEPENDENCIES=ON"
# Passed to make compilation command.
- MAKE_COMPILE_ARGS="-j$(nproc)"
+ # Concurrency was determined after some trial and error. Higher
+ # values resulted in memory limit issues and compiler crashes.
+ MAKE_COMPILE_ARGS="-j4"
# shellcheck disable=SC2034
# Expected by Yetus for compiling non-jvm projects.
JVM_REQUIRED=false
@@ -91,7 +93,10 @@ function personality_globals
PROCLIMIT=10000
# Override if you want to bump up the memlimit for docker.
# shellcheck disable=SC2034
- DOCKERMEMLIMIT=4g
+ DOCKERMEMLIMIT=16g
+ # shellcheck disable=SC2034
+ # Logs the ctest output when a test fails.
+ CTEST_OUTPUT_ON_FAILURE=1
}
## @description Queue up modules for this personality