[impala] branch master updated: IMPALA-8147: part 1/2: make make_*.sh redundant.

[ta...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git


The following commit(s) were added to refs/heads/master by this push:
     new d0c1030  IMPALA-8147: part 1/2: make make_*.sh redundant.
d0c1030 is described below

commit d0c103095019f1b80dec63193847eb516d7f8dd7
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Wed Jan 30 07:01:33 2019 -0800

    IMPALA-8147: part 1/2: make make_*.sh redundant.
    
    The logic in that file, which is mostly about constructing argument
    lists for CMake and make, is moved to functions in buildall.sh.
    
    A new option -release_and_debug is added to buildall.sh to build
    both the debug and release builds. This is convenient for building
    a binary Impala for distribution because you want to have both
    sets of binaries available.
    
    make*.sh are not yet removed in order to make the transition easier.
    
    Testing:
    Ran buildall.sh locally with -release_and_debug, confirmed that
    all of the right binaries were generated.
    
    Change-Id: I70e4f65712166348ca006bc68e1a1e18e853d3a0
    Reviewed-on: http://gerrit.cloudera.org:8080/12368
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/common/CMakeLists.txt |   4 ++
 bin/distcc/README.md         |   7 +--
 bin/make_impala.sh           |   3 ++
 buildall.sh                  | 100 +++++++++++++++++++++++++++++++++++--------
 4 files changed, 89 insertions(+), 25 deletions(-)

diff --git a/be/src/common/CMakeLists.txt b/be/src/common/CMakeLists.txt
index daa274d..2ea3b0c 100644
--- a/be/src/common/CMakeLists.txt
+++ b/be/src/common/CMakeLists.txt
@@ -36,12 +36,16 @@ add_library(Common
   ${VERSION_CC_GEN_OUTPUT}
 )
 
+# Command to generate the build version file if not present. We don't automatically
+# regenerate the file if present, which speeds up incremental builds but can lead
+# to the version being stale.
 add_custom_command(
   OUTPUT ${VERSION_CC_GEN_OUTPUT}
   COMMAND $ENV{IMPALA_HOME}/bin/gen_build_version.py
   COMMENT "Generating the version.cc file"
   VERBATIM
 )
+
 add_dependencies(Common gen-deps)
 
 add_library(GlobalFlags
diff --git a/bin/distcc/README.md b/bin/distcc/README.md
index a8487b2..180df0d 100644
--- a/bin/distcc/README.md
+++ b/bin/distcc/README.md
@@ -56,10 +56,6 @@ you can either
 ```
 make -j$(distcc -j)
 ```
-or
-```
-bin/make_impala.sh
-```
 
 ### Switching back to local compilation
 If you want to compile a very small change, a local build might be faster.
@@ -71,8 +67,7 @@ to switch back
 switch_compiler distcc
 ```
 ### Second time
-If you open a new terminal and attempt to build with "make" or "bin/make_impala.sh",
-that will fail. To fix:
+If you open a new terminal and attempt to build with "make" that will fail. To fix:
 ```
 source "$IMPALA_HOME"/bin/impala-config.sh   # Skip if already done
 source "$IMPALA_HOME"/bin/distcc/distcc_env.sh
diff --git a/bin/make_impala.sh b/bin/make_impala.sh
index 8ac7639..2ce837f 100755
--- a/bin/make_impala.sh
+++ b/bin/make_impala.sh
@@ -103,6 +103,9 @@ do
   esac
 done
 
+echo "IMPALA-8147: make_impala.sh, make_debug.sh, make_release.sh and make_asan.sh "
+echo "are deprecated. Please migrate your scripts to use buildall.sh or make directly"
+
 if [ $BUILD_TESTS -eq 1 ]; then
   MAKE_TARGETS+=" be-test be-benchmarks"
 fi
diff --git a/buildall.sh b/buildall.sh
index bfe3507..8a182a6 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -60,17 +60,19 @@ START_IMPALA_CLUSTER=0
 IMPALA_KERBERIZE=0
 SNAPSHOT_FILE=
 METASTORE_SNAPSHOT_FILE=
-MAKE_IMPALA_ARGS=""
 CODE_COVERAGE=0
 BUILD_ASAN=0
 BUILD_FE_ONLY=0
+BUILD_TESTS=1
+GEN_CMAKE_ONLY=0
+BUILD_RELEASE_AND_DEBUG=0
 BUILD_TIDY=0
 BUILD_UBSAN=0
 BUILD_UBSAN_FULL=0
 BUILD_TSAN=0
+BUILD_SHARED_LIBS=0
 # Export MAKE_CMD so it is visible in scripts that invoke make, e.g. copy-udfs-udas.sh
 export MAKE_CMD=make
-LZO_CMAKE_ARGS=
 
 # Defaults that can be picked up from the environment, but are overridable through the
 # commandline.
@@ -91,11 +93,11 @@ do
       TESTS_ACTION=0
       ;;
     -build_shared_libs|-so)
-      MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_shared_libs"
+      BUILD_SHARED_LIBS=1
       ;;
     -notests)
       TESTS_ACTION=0
-      MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -notests"
+      BUILD_TESTS=0
       ;;
     -format)
       FORMAT_CLUSTER=1
@@ -114,6 +116,9 @@ do
     -release)
       CMAKE_BUILD_TYPE=Release
       ;;
+    -release_and_debug)
+      BUILD_RELEASE_AND_DEBUG=1
+      ;;
     -codecoverage)
       CODE_COVERAGE=1
       ;;
@@ -183,12 +188,10 @@ do
       BUILD_FE_ONLY=1
       ;;
     -ninja)
-      MAKE_IMPALA_ARGS+=" -ninja"
-      LZO_CMAKE_ARGS+=" -GNinja"
       MAKE_CMD=ninja
       ;;
     -cmake_only)
-      MAKE_IMPALA_ARGS+=" -cmake_only"
+      GEN_CMAKE_ONLY=1
       ;;
     -help|*)
       echo "buildall.sh - Builds Impala and runs all tests."
@@ -199,6 +202,8 @@ do
       echo "[-format_cluster] : Format the minicluster [Default: False]"
       echo "[-format_metastore] : Format the metastore db [Default: False]"
       echo "[-format_sentry_policy_db] : Format the Sentry policy db [Default: False]"
+      echo "[-release_and_debug] : Build both release and debug binaries. Overrides "\
+           "other build types [Default: false]"
       echo "[-release] : Release build [Default: debug]"
       echo "[-codecoverage] : Build with code coverage [Default: False]"
       echo "[-asan] : Address sanitizer build [Default: False]"
@@ -297,8 +302,6 @@ if [[ ${BUILD_TSAN} -eq 1 ]]; then
   CMAKE_BUILD_TYPE=TSAN
 fi
 
-MAKE_IMPALA_ARGS+=" -build_type=${CMAKE_BUILD_TYPE}"
-
 # If we aren't kerberized then we certainly don't need to talk about
 # re-sourcing impala-config.
 if [[ ${IMPALA_KERBERIZE} -eq 0 ]]; then
@@ -370,21 +373,67 @@ bootstrap_dependencies() {
 
 # Build the Impala frontend and its dependencies.
 build_fe() {
-  "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -fe_only
+  generate_cmake_files $CMAKE_BUILD_TYPE
+  ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} fe
 }
 
-# Build all components.
+# Build all components. The build type is specified as the first argument, and the
+# second argument is 0 if targets that are independent of the build type (like the
+# frontend) should not be built or non-zero otherwise. E.g. to build DEBUG including
+# build-type-independent artifacts.
+#   build_all_components DEBUG 1
 build_all_components() {
+  build_type=$1
+  build_independent_targets=$2
   echo ">>> Building all components"
-  # Build the Impala frontend, backend and external data source API.
-  MAKE_IMPALA_ARGS+=" -fe -cscope -tarballs"
-  if [[ -e "$IMPALA_LZO" ]]
-  then
-    MAKE_IMPALA_ARGS+=" -impala-lzo"
+  generate_cmake_files $build_type
+
+  # Force regenerating the build version and timestamp (this doesn't happen automatically
+  # in incremental builds).
+  $IMPALA_HOME/bin/gen_build_version.py
+
+  # If we skip specifying targets, everything we need gets built.
+  local MAKE_TARGETS=""
+  if [[ $BUILD_TESTS -eq 0 ]]; then
+    # Specify all the non-test targets
+    MAKE_TARGETS="impalad statestored catalogd fesupport loggingsupport ImpalaUdf \
+        udasample udfsample"
+    if (( build_independent_targets )); then
+      MAKE_TARGETS+=" cscope fe tarballs"
+    fi
+    if [[ -e "$IMPALA_LZO" ]]; then
+      MAKE_TARGETS+=" impala-lzo"
+    fi
   fi
+  ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} ${IMPALA_MAKE_FLAGS} ${MAKE_TARGETS}
+}
+
 
-  echo "Running make_impala.sh ${MAKE_IMPALA_ARGS}"
-  "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
+# Called with the CMAKE_BUILD_TYPE as the first argument, e.g.
+#   generate_cmake_files DEBUG
+generate_cmake_files() {
+  local build_type=$1
+  echo ">>> Generating CMake files" "CMAKE_BUILD_TYPE=$build_type"\
+       "BUILD_SHARED_LIBS=$BUILD_SHARED_LIBS" "MAKE_CMD=$MAKE_CMD"
+  # Remove cache to ensure that any changes to cmake arguments take effect.
+  rm -f ./CMakeCache.txt
+  local CMAKE_ARGS=(-DCMAKE_BUILD_TYPE=${build_type})
+  if [[ $BUILD_SHARED_LIBS -eq 1 ]]; then
+    CMAKE_ARGS+=(-DBUILD_SHARED_LIBS=ON)
+  fi
+  if [[ "${MAKE_CMD}" = "ninja" ]]; then
+    CMAKE_ARGS+=(-GNinja)
+  fi
+  if [[ ("$build_type" == "ADDRESS_SANITIZER") \
+            || ("$build_type" == "TIDY") \
+            || ("$build_type" == "UBSAN") \
+            || ("$build_type" == "UBSAN_FULL") \
+            || ("$build_type" == "TSAN") ]]; then
+    CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/clang_toolchain.cmake)
+  else
+    CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/toolchain.cmake)
+  fi
+  cmake . ${CMAKE_ARGS[@]}
 }
 
 # Do any configuration of the test cluster required by the script arguments.
@@ -492,7 +541,20 @@ if [[ "$BUILD_FE_ONLY" -eq 1 ]]; then
   exit 0
 fi
 
-build_all_components
+if [[ "$GEN_CMAKE_ONLY" -eq 1 ]]; then
+  generate_cmake_files $CMAKE_BUILD_TYPE
+  exit 0
+fi
+if [[ "$BUILD_RELEASE_AND_DEBUG" -eq 1 ]]; then
+  # Build the standard release and debug builds. We can't do this for arbitrary build
+  # types because many build types reuse the same be/build/debug and be/build/release
+  # trees.
+  build_all_components RELEASE 1
+  # Avoid rebuilding targets that are independent of the build type.
+  build_all_components DEBUG 0
+else
+  build_all_components $CMAKE_BUILD_TYPE 1
+fi
 
 if [[ $NEED_MINICLUSTER -eq 1 ]]; then
   reconfigure_test_cluster