You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/05/12 22:09:48 UTC
[13/50] [abbrv] incubator-impala git commit: Add ninja support for
faster incremental builds
Add ninja support for faster incremental builds
Ninja resolves dependencies much faster, so if only a couple of files
are changed "ninja -j ${IMPALA_BUILD_THREADS} impalad" returns within a
second or two, while make can take tens of seconds to resolve all the
dependencies.
This requires ninja to be installed. It is widely available, e.g. in the
ninja-build package on Ubuntu.
Ninja can be enabled by passing "-ninja" to buildall.sh or
make_impala.sh. The same targets should work as with make.
The default Ninja status output is fairly terse. It can be customised
with an environment variable. E.g. I have
export NINJA_STATUS="[%u to run/%r running/%f finished] "
Also fixes a bug in make_impala.sh where invalid arguments were ignored.
Change-Id: I2cea479615fe850c98d30110de043ecb6358dcda
Reviewed-on: http://gerrit.cloudera.org:8080/2923
Tested-by: Internal Jenkins
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/6e89f1a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/6e89f1a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/6e89f1a2
Branch: refs/heads/master
Commit: 6e89f1a250ea1dd16a98f11dea2cdfa68e6dc73a
Parents: c9df348
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Mon May 2 09:23:39 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Thu May 12 14:17:53 2016 -0700
----------------------------------------------------------------------
bin/make_impala.sh | 33 ++++++++++++++++++++-------------
buildall.sh | 17 ++++++++++++-----
2 files changed, 32 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e89f1a2/bin/make_impala.sh
----------------------------------------------------------------------
diff --git a/bin/make_impala.sh b/bin/make_impala.sh
index 6a43251..70da52b 100755
--- a/bin/make_impala.sh
+++ b/bin/make_impala.sh
@@ -25,6 +25,7 @@ CLEAN=0
TARGET_BUILD_TYPE=${TARGET_BUILD_TYPE:-""}
BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS:-""}
CMAKE_ONLY=0
+MAKE_CMD=make
# parse command line options
for ARG in $*
@@ -45,22 +46,26 @@ do
-build_static_libs)
BUILD_SHARED_LIBS="OFF"
;;
+ -ninja)
+ MAKE_CMD=ninja
+ ;;
-cmake_only)
CMAKE_ONLY=1
;;
- -help)
+ -help|*)
echo "make_impala.sh [-build_type=<build type> -notests -clean]"
echo "[-build_type] : Target build type. Examples: Debug, Release, Address_sanitizer."
echo " If omitted, the last build target is built incrementally"
echo "[-build_shared_libs] : Link all executables dynamically"
echo "[-build_static_libs] : Link all executables statically (the default)"
echo "[-cmake_only] : Generate makefiles and exit"
+ echo "[-ninja] : Use the Ninja build tool instead of Make"
echo "[-notests] : Omits building the tests."
echo "[-clean] : Cleans previous build artifacts."
echo ""
echo "If either -build_type or -build_*_libs is set, cmake will be re-run for the "
echo "project. Otherwise the last cmake configuration will continue to take effect."
- exit
+ exit 1
;;
esac
done
@@ -87,7 +92,8 @@ echo "**************************************************************************
cd ${IMPALA_HOME}
-if [ "x${TARGET_BUILD_TYPE}" != "x" ] || [ "x${BUILD_SHARED_LIBS}" != "x" ]
+if [ "x${TARGET_BUILD_TYPE}" != "x" ] || [ "x${BUILD_SHARED_LIBS}" != "x" ] || \
+ [ "${MAKE_CMD}" != "make" ]
then
rm -f ./CMakeCache.txt
CMAKE_ARGS=()
@@ -99,6 +105,10 @@ then
CMAKE_ARGS+=(-DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS})
fi
+ if [ "${MAKE_CMD}" = "ninja" ]; then
+ CMAKE_ARGS+=" -GNinja"
+ fi
+
if [[ ! -z $IMPALA_TOOLCHAIN ]]; then
if [[ "$TARGET_BUILD_TYPE" == "ADDRESS_SANITIZER" ]]; then
@@ -113,7 +123,7 @@ fi
if [ $CLEAN -eq 1 ]
then
- make clean
+ ${MAKE_CMD} clean
fi
$IMPALA_HOME/bin/gen_build_version.py --noclean
@@ -122,21 +132,18 @@ if [ $CMAKE_ONLY -eq 1 ]; then
exit 0
fi
-cd $IMPALA_HOME/common/function-registry
-make
-
-cd $IMPALA_HOME
+${MAKE_CMD} function-registry
# With parallelism, make doesn't always make statestored and catalogd correctly if you
# write make -jX impalad statestored catalogd. So we keep them separate and after impalad,
# which they link to.
-make -j${IMPALA_BUILD_THREADS:-4} impalad
+${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} impalad
-make statestored
-make catalogd
+${MAKE_CMD} statestored
+${MAKE_CMD} catalogd
if [ $BUILD_TESTS -eq 1 ]
then
- make -j${IMPALA_BUILD_THREADS:-4}
+ ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4}
else
- make -j${IMPALA_BUILD_THREADS:-4} fesupport loggingsupport ImpalaUdf
+ ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} fesupport loggingsupport ImpalaUdf
fi
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/6e89f1a2/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index fa16613..08a3ceb 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -49,6 +49,8 @@ MAKE_IMPALA_ARGS=""
BUILD_COVERAGE=0
BUILD_ASAN=0
BUILD_FE_ONLY=0
+MAKE_CMD=make
+LZO_CMAKE_ARGS=
# Defaults that can be picked up from the environment, but are overridable through the
# commandline.
@@ -143,6 +145,11 @@ do
-fe_only)
BUILD_FE_ONLY=1
;;
+ -ninja)
+ MAKE_IMPALA_ARGS+=" -ninja"
+ LZO_CMAKE_ARGS+=" -GNinja"
+ MAKE_CMD=ninja
+ ;;
-help|*)
echo "buildall.sh - Builds Impala and runs all tests."
echo "[-noclean] : Omits cleaning all packages before building. Will not kill"\
@@ -253,7 +260,7 @@ MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_type=${CMAKE_BUILD_TYPE}"
if [ $BUILD_FE_ONLY -eq 1 ]; then
$IMPALA_HOME/bin/make_impala.sh ${MAKE_IMPALA_ARGS} -cmake_only
- make fe
+ ${MAKE_CMD} fe
exit 0
fi
@@ -304,11 +311,11 @@ if [ -e $IMPALA_LZO ]
then
pushd $IMPALA_LZO
if [[ ! -z $IMPALA_TOOLCHAIN ]]; then
- cmake -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake
- else
- cmake .
+ LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
fi
- make
+ rm -f CMakeCache.txt
+ cmake ${LZO_CMAKE_ARGS}
+ ${MAKE_CMD}
popd
fi