You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2018/05/24 17:46:53 UTC
[2/5] impala git commit: Trimming build-all-flag-combinations and
adding minicluster profile.
Trimming build-all-flag-combinations and adding minicluster profile.
build-all-flag-combinations.sh is the long pole in pre-commit testing.
It takes about 2.5 hours and runs about 40 builds per the following.
$ curl --silent https://jenkins.impala.io/job/all-build-options-ub1604/1707/consoleText | grep 'Building with OPTIONS' | wc -l
40
This commit changes this to run only 7 builds, exercising every path,
but not every combination of paths. The paths exercised are:
Options -skiptests -noclean and profile 3
Options -skiptests -noclean -so and profile 2
Options -skiptests -noclean -release and profile 3
Options -skiptests -noclean -release -so -ninja and profile 3
Options -skiptests -noclean -asan and profile 3
Options -skiptests -noclean -ubsan -so -ninja and profile 2
Options -skiptests -noclean -tsan and profile 3
I've also added explicitly building with both minicluster profiles
and a trivial dryrun option to print the above.
The options are simply hard-coded rather than being clever
and generating them. We decided this was easier to deal with.
Change-Id: I258732a3d963958f76d99f9d7b51450ed67bec21
Reviewed-on: http://gerrit.cloudera.org:8080/10489
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/3bedfcdb
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/3bedfcdb
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/3bedfcdb
Branch: refs/heads/master
Commit: 3bedfcdbfe90a89cae687a1498c926641a2b656e
Parents: 879f106
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Wed May 23 11:41:44 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu May 24 00:30:36 2018 +0000
----------------------------------------------------------------------
bin/jenkins/build-all-flag-combinations.sh | 80 +++++++++++++++----------
1 file changed, 50 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/3bedfcdb/bin/jenkins/build-all-flag-combinations.sh
----------------------------------------------------------------------
diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh
index ef1a89e..841b668 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -17,45 +17,65 @@
# specific language governing permissions and limitations
# under the License.
-# Build Impala with the most common build configurations and check that the build
-# succeeds.a Intended for use as a precommit test to make sure nothing got broken.
+# Build Impala with the a variety of common build configurations and check that the build
+# succeeds. Intended for use as a precommit test to make sure nothing got broken.
#
# Assumes that ninja and ccache are installed.
+#
+# Usage: build-all-flag-combinations.sh [--dryrun]
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
. bin/impala-config.sh
-OPTIONS=("-skiptests" "-noclean")
-FAILED_OPTIONS=""
-for BUILD_TYPE in "" -asan -release -ubsan -tsan
-do
- OPTIONS[2]=$BUILD_TYPE
- for NINJA in "" -ninja
- do
- OPTIONS[3]=$NINJA
- for BUILD_SHARED_LIBS in "" -so
- do
- OPTIONS[4]=$BUILD_SHARED_LIBS
- if ! ./bin/clean.sh
- then
- echo "Clean failed"
- exit 1
- fi
- echo "Building with OPTIONS: ${OPTIONS[@]}"
- if ! time -p ./buildall.sh ${OPTIONS[@]}
- then
- echo "Build failed with OPTIONS: ${OPTIONS[@]}"
- FAILED_OPTIONS="${FAILED_OPTIONS}:${OPTIONS[@]}"
- fi
- ccache -s
- done
- done
+
+# These are configurations for buildall, with a special sigil for
+# "minicluster profile" where appropriate.
+CONFIGS=(
+ # Test gcc builds with and without -so:
+ "-skiptests -noclean"
+ "-skiptests -noclean -so -profile2"
+ "-skiptests -noclean -release"
+ "-skiptests -noclean -release -so -ninja"
+ # clang sanitizer builds:
+ "-skiptests -noclean -asan"
+ "-skiptests -noclean -ubsan -so -ninja -profile2"
+ "-skiptests -noclean -tsan"
+)
+
+FAILED=""
+
+for CONFIG in "${CONFIGS[@]}"; do
+ CONFIG2=${CONFIG/-profile2/}
+ if [[ "$CONFIG" != "$CONFIG2" ]]; then
+ CONFIG=$CONFIG2
+ export IMPALA_MINICLUSTER_PROFILE_OVERRIDE=2
+ else
+ export IMPALA_MINICLUSTER_PROFILE_OVERRIDE=3
+ fi
+
+ DESCRIPTION="Options $CONFIG and profile $IMPALA_MINICLUSTER_PROFILE_OVERRIDE"
+
+ if [[ $# == 1 && $1 == "--dryrun" ]]; then
+ echo $DESCRIPTION
+ continue
+ fi
+
+ if ! ./bin/clean.sh; then
+ echo "Clean failed"
+ exit 1
+ fi
+ echo "Building with OPTIONS: $DESCRIPTION"
+ if ! time -p ./buildall.sh $CONFIG; then
+ echo "Build failed: $DESCRIPTION"
+ FAILED="${FAILED}:${DESCRIPTION}"
+ fi
+ ccache -s
done
-if [[ "$FAILED_OPTIONS" != "" ]]
+if [[ "$FAILED" != "" ]]
then
- echo "Builds with the following options failed:"
- echo "$FAILED_OPTIONS"
+ echo "The following builds failed:"
+ echo "$FAILED"
exit 1
fi