You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/12/15 23:01:10 UTC
[44/50] [abbrv] incubator-impala git commit: Add all build targets to
CMake and speed up builds
Add all build targets to CMake and speed up builds
Use CMake's dependency resolution always instead of serial execution of
targets via shell scripts. This improves parallelism by building fe,
be, and other targets at the same time and avoid some overhead from
invoking "make" multiple times. This reduces the time taken for
an incremental compilation of fe and be from 56s to 24s with this
command:
./buildall.sh -debug -noclean -notests -skiptests -ninja
Also use Impala-lzo's build script. This depends on the IMPALA-4277
fixes to the Impala-lzo build script.
Log directory creation is also moved from impala-config.sh to
buildall.sh. This means that impala-config.sh has no side-effects and
can be run concurrently with no issues.
Also make sure that "make" builds all the same artifacts as buildall.sh
when run with no args.
Testing:
Ran a jenkins core job, also experimented locally. Ran a jenkins core
job with distcc disabled - this exposed some concurrency bugs where
impala-config.sh fails if run concurrently.
Change-Id: I23617adf13bdeb034c24f6bba14b5ae480e8dd26
Reviewed-on: http://gerrit.cloudera.org:8080/4790
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/fc4ee65f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/fc4ee65f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/fc4ee65f
Branch: refs/heads/hadoop-next
Commit: fc4ee65f9f51581915df55bfa07aaacc6eb610d0
Parents: e6ef3b4
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Oct 20 09:23:15 2016 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Dec 14 23:42:19 2016 +0000
----------------------------------------------------------------------
CMakeLists.txt | 21 ++++++-------
be/src/benchmarks/CMakeLists.txt | 4 +++
be/src/service/CMakeLists.txt | 4 +--
bin/impala-config.sh | 15 +---------
bin/make_impala.sh | 55 +++++++++++++++++++++++++----------
buildall.sh | 48 +++++++++++++++---------------
ext-data-source/CMakeLists.txt | 2 +-
fe/CMakeLists.txt | 2 +-
8 files changed, 85 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index dff59ad..661c9a6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -305,21 +305,22 @@ add_subdirectory(be)
add_subdirectory(fe)
add_subdirectory(ext-data-source)
-# Run FE and BE tests
-add_custom_target(testall
- COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_SOURCE_DIR}/fe mvn test
- COMMAND ${CMAKE_SOURCE_DIR}/bin/runbackendtests.sh
-)
+add_custom_target(tarballs ALL DEPENDS shell_tarball)
-# Load test data
-add_custom_target(loadtestdata
- COMMAND ${CMAKE_SOURCE_DIR}/bin/load-test-data.sh
+add_custom_target(shell_tarball DEPENDS thrift-deps
+ COMMAND "${CMAKE_SOURCE_DIR}/shell/make_shell_tarball.sh"
)
-add_custom_target(benchmark_run
- COMMAND ${CMAKE_SOURCE_DIR}/be/bin/run_hive_benchmark.py
+add_custom_target(cscope ALL
+ COMMAND "${CMAKE_SOURCE_DIR}/bin/gen-cscope.sh"
)
+if (DEFINED ENV{IMPALA_LZO} AND EXISTS $ENV{IMPALA_LZO})
+ add_custom_target(impala-lzo ALL DEPENDS thrift-deps
+ COMMAND $ENV{IMPALA_LZO}/build.sh ${CMAKE_SOURCE_DIR} $ENV{IMPALA_TOOLCHAIN}
+ )
+endif()
+
# Dump include paths to a file
if (DUMP_INCLUDE_PATHS)
file(REMOVE "${DUMP_INCLUDE_PATHS}")
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/be/src/benchmarks/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/CMakeLists.txt b/be/src/benchmarks/CMakeLists.txt
index ba8bcfc..5d3dfbd 100644
--- a/be/src/benchmarks/CMakeLists.txt
+++ b/be/src/benchmarks/CMakeLists.txt
@@ -22,9 +22,13 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/benchmarks")
# where to put generated binaries
set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/benchmarks")
+# Add custom target to only build the backend benchmarks
+add_custom_target(be-benchmarks)
+
FUNCTION(ADD_BE_BENCHMARK BENCHMARK_NAME)
ADD_EXECUTABLE(${BENCHMARK_NAME} ${BENCHMARK_NAME}.cc)
TARGET_LINK_LIBRARIES(${BENCHMARK_NAME} ${IMPALA_LINK_LIBS})
+ ADD_DEPENDENCIES(be-benchmarks ${BENCHMARK_NAME})
ENDFUNCTION()
ADD_BE_BENCHMARK(atod-benchmark)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/be/src/service/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt
index 6d5a897..c886bf4 100644
--- a/be/src/service/CMakeLists.txt
+++ b/be/src/service/CMakeLists.txt
@@ -53,11 +53,11 @@ add_executable(impalad
# All Impala daemons run from the same binary. The code that is run is determined by the
# name (i.e. argv[0]) of the command that executes the binary, so we create symlinks for
# statestored and catalogd.
-ADD_CUSTOM_TARGET(statestored
+ADD_CUSTOM_TARGET(statestored ALL
${CMAKE_COMMAND} -E create_symlink ${EXECUTABLE_OUTPUT_PATH}/impalad
${BUILD_OUTPUT_ROOT_DIRECTORY}/statestore/statestored DEPENDS impalad)
-ADD_CUSTOM_TARGET(catalogd
+ADD_CUSTOM_TARGET(catalogd ALL
${CMAKE_COMMAND} -E create_symlink ${EXECUTABLE_OUTPUT_PATH}/impalad
${BUILD_OUTPUT_ROOT_DIRECTORY}/catalog/catalogd DEPENDS impalad)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index aad231c..2a0ac83 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -228,25 +228,12 @@ export IMPALA_FE_TEST_COVERAGE_DIR="${IMPALA_FE_TEST_LOGS_DIR}/coverage"
export IMPALA_BE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/be_tests"
export IMPALA_EE_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/ee_tests"
export IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR="${IMPALA_LOGS_DIR}/custom_cluster_tests"
-# List of all Impala log dirs and create them.
+# List of all Impala log dirs so they can be created by buildall.sh
export IMPALA_ALL_LOGS_DIRS="${IMPALA_CLUSTER_LOGS_DIR}
${IMPALA_DATA_LOADING_LOGS_DIR} ${IMPALA_DATA_LOADING_SQL_DIR}
${IMPALA_EE_TEST_LOGS_DIR} ${IMPALA_FE_TEST_COVERAGE_DIR}
${IMPALA_BE_TEST_LOGS_DIR} ${IMPALA_EE_TEST_LOGS_DIR}
${IMPALA_CUSTOM_CLUSTER_TEST_LOGS_DIR}"
-mkdir -p $IMPALA_ALL_LOGS_DIRS
-
-# Create symlinks Testing/Temporary and be/Testing/Temporary that point to the BE test
-# log dir to capture the all logs of BE unit tests. Gtest has Testing/Temporary
-# hardwired in its code, so we cannot change the output dir by configuration.
-# We create two symlinks to capture the logs when running ctest either from
-# ${IMPALA_HOME} or ${IMPALA_HOME}/be.
-rm -rf "${IMPALA_HOME}/Testing"
-mkdir -p "${IMPALA_HOME}/Testing"
-ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/Testing/Temporary"
-rm -rf "${IMPALA_HOME}/be/Testing"
-mkdir -p "${IMPALA_HOME}/be/Testing"
-ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/be/Testing/Temporary"
# Reduce the concurrency for local tests to half the number of cores in the system.
# Note than nproc may not be available on older distributions (centos5.5)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/bin/make_impala.sh
----------------------------------------------------------------------
diff --git a/bin/make_impala.sh b/bin/make_impala.sh
index c198378..ec8421a 100755
--- a/bin/make_impala.sh
+++ b/bin/make_impala.sh
@@ -17,19 +17,25 @@
# specific language governing permissions and limitations
# under the License.
-# Incrementally compiles the BE.
+# Incrementally compiles the frontend and backend.
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
: ${IMPALA_TOOLCHAIN=}
+BUILD_EVERYTHING=1
+BUILD_FE_ONLY=0
BUILD_TESTS=1
CLEAN=0
TARGET_BUILD_TYPE=${TARGET_BUILD_TYPE:-""}
BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS:-""}
CMAKE_ONLY=0
MAKE_CMD=make
+MAKE_ARGS=-j${IMPALA_BUILD_THREADS:-4}
+
+# The minimal make targets if BUILD_EVERYTHING is 0.
+MAKE_TARGETS="impalad statestored catalogd fesupport loggingsupport ImpalaUdf"
# parse command line options
for ARG in $*
@@ -37,6 +43,7 @@ do
case "$ARG" in
-notests)
BUILD_TESTS=0
+ BUILD_EVERYTHING=0
;;
-clean)
CLEAN=1
@@ -56,6 +63,22 @@ do
-cmake_only)
CMAKE_ONLY=1
;;
+ -fe)
+ MAKE_TARGETS+=" fe"
+ ;;
+ -fe_only)
+ BUILD_FE_ONLY=1
+ BUILD_EVERYTHING=0
+ ;;
+ -cscope)
+ MAKE_TARGETS+=" cscope"
+ ;;
+ -impala-lzo)
+ MAKE_TARGETS+=" impala-lzo"
+ ;;
+ -tarballs)
+ MAKE_TARGETS+=" tarballs"
+ ;;
-help|*)
echo "make_impala.sh [-build_type=<build type> -notests -clean]"
echo "[-build_type] : Target build type. Examples: Debug, Release, Address_sanitizer."
@@ -63,9 +86,14 @@ do
echo "[-build_shared_libs] : Link all executables dynamically"
echo "[-build_static_libs] : Link all executables statically (the default)"
echo "[-cmake_only] : Generate makefiles and exit"
+ echo "[-fe] : Builds fe in addition to backend."
+ echo "[-fe_only] : Builds fe only."
echo "[-ninja] : Use the Ninja build tool instead of Make"
- echo "[-notests] : Omits building the tests."
+ echo "[-notests] : Omits building the backend tests and benchmarks."
echo "[-clean] : Cleans previous build artifacts."
+ echo "[-cscope] : Builds cscope metadata."
+ echo "[-impala-lzo] : Builds Impala LZO."
+ echo "[-tarballs] : Builds additional tarballs like the shell tarball."
echo ""
echo "If either -build_type or -build_*_libs is set, cmake will be re-run for the "
echo "project. Otherwise the last cmake configuration will continue to take effect."
@@ -74,6 +102,10 @@ do
esac
done
+if [ $BUILD_TESTS -eq 1 ]; then
+ MAKE_TARGETS+=" be-test be-benchmarks"
+fi
+
echo "********************************************************************************"
echo " Building Impala "
if [ "x${TARGET_BUILD_TYPE}" != "x" ];
@@ -138,18 +170,11 @@ if [ $CMAKE_ONLY -eq 1 ]; then
exit 0
fi
-${MAKE_CMD} function-registry
-
-# With parallelism, make doesn't always make statestored and catalogd correctly if you
-# write make -jX impalad statestored catalogd. So we keep them separate and after impalad,
-# which they link to.
-${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} impalad
-
-${MAKE_CMD} statestored
-${MAKE_CMD} catalogd
-if [ $BUILD_TESTS -eq 1 ]
-then
- ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4}
+MAKE_ARGS=-j${IMPALA_BUILD_THREADS:-4}
+if [ $BUILD_FE_ONLY -eq 1 ]; then
+ ${MAKE_CMD} ${MAKE_ARGS} fe
+elif [ $BUILD_EVERYTHING -eq 1 ]; then
+ ${MAKE_CMD} ${MAKE_ARGS}
else
- ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} fesupport loggingsupport ImpalaUdf
+ ${MAKE_CMD} ${MAKE_ARGS} ${MAKE_TARGETS}
fi
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index a4791b7..fc66acb 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -286,6 +286,23 @@ if [[ $TESTS_ACTION -eq 1 || $TESTDATA_ACTION -eq 1 || $FORMAT_CLUSTER -eq 1 ||
NEED_MINICLUSTER=1
fi
+create_log_dirs() {
+ # Create all of the log directories.
+ mkdir -p $IMPALA_ALL_LOGS_DIRS
+
+ # Create symlinks Testing/Temporary and be/Testing/Temporary that point to the BE test
+ # log dir to capture the all logs of BE unit tests. Gtest has Testing/Temporary
+ # hardwired in its code, so we cannot change the output dir by configuration.
+ # We create two symlinks to capture the logs when running ctest either from
+ # ${IMPALA_HOME} or ${IMPALA_HOME}/be.
+ rm -rf "${IMPALA_HOME}/Testing"
+ mkdir -p "${IMPALA_HOME}/Testing"
+ ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/Testing/Temporary"
+ rm -rf "${IMPALA_HOME}/be/Testing"
+ mkdir -p "${IMPALA_HOME}/be/Testing"
+ ln -fs "${IMPALA_BE_TEST_LOGS_DIR}" "${IMPALA_HOME}/be/Testing/Temporary"
+}
+
bootstrap_dependencies() {
# Populate necessary thirdparty components unless it's set to be skipped.
if [[ "${SKIP_TOOLCHAIN_BOOTSTRAP}" = true ]]; then
@@ -310,37 +327,20 @@ bootstrap_dependencies() {
# Build the Impala frontend and its dependencies.
build_fe() {
- "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -cmake_only
- "${MAKE_CMD}" fe
+ "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -fe_only
}
# Build all components.
build_all_components() {
- # Build common and backend. This also sets up the CMake files.
- echo "Calling make_impala.sh ${MAKE_IMPALA_ARGS}"
- "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
-
+ # Build the Impala frontend, backend and external data source API.
+ MAKE_IMPALA_ARGS+=" -fe -cscope -tarballs"
if [[ -e "$IMPALA_LZO" ]]
then
- pushd "$IMPALA_LZO"
- LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
- rm -f CMakeCache.txt
- cmake ${LZO_CMAKE_ARGS}
- "${MAKE_CMD}"
- popd
+ MAKE_IMPALA_ARGS+=" -impala-lzo"
fi
- # Build the Java components (fe and external data source API).
- pushd "$IMPALA_HOME"
- "${MAKE_CMD}" ext-data-source fe
- popd
-
- # Build the shell tarball
- echo "Creating shell tarball"
- "${IMPALA_HOME}/shell/make_shell_tarball.sh"
-
- # Generate list of files for Cscope to index
- "$IMPALA_HOME/bin/gen-cscope.sh"
+ echo "Running make_impala.sh ${MAKE_IMPALA_ARGS}"
+ "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
}
# Do any configuration of the test cluster required by the script arguments.
@@ -426,6 +426,8 @@ if [[ "$CLEAN_ACTION" -eq 1 ]]; then
"$IMPALA_HOME/bin/clean.sh"
fi
+create_log_dirs
+
bootstrap_dependencies
if [[ "$BUILD_FE_ONLY" -eq 1 ]]; then
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/ext-data-source/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext-data-source/CMakeLists.txt b/ext-data-source/CMakeLists.txt
index d2c1077..1a53278 100644
--- a/ext-data-source/CMakeLists.txt
+++ b/ext-data-source/CMakeLists.txt
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-add_custom_target(ext-data-source DEPENDS thrift-deps
+add_custom_target(ext-data-source ALL DEPENDS thrift-deps
COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/fc4ee65f/fe/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/fe/CMakeLists.txt b/fe/CMakeLists.txt
index c162435..e06f467 100644
--- a/fe/CMakeLists.txt
+++ b/fe/CMakeLists.txt
@@ -15,6 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-add_custom_target(fe DEPENDS thrift-deps function-registry ext-data-source
+add_custom_target(fe ALL DEPENDS thrift-deps function-registry ext-data-source
COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
)