You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2016/04/20 01:19:33 UTC
yetus git commit: YETUS-378. flink, hbase,
jmeter personalities custom tests need to be made qbt aware
Repository: yetus
Updated Branches:
refs/heads/YETUS-156 3f782d346 -> 64377ca5a
YETUS-378. flink, hbase, jmeter personalities custom tests need to be made qbt aware
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/64377ca5
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/64377ca5
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/64377ca5
Branch: refs/heads/YETUS-156
Commit: 64377ca5a0bacf3f6ed946c2507018dc02d0975e
Parents: 3f782d3
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Apr 18 14:18:15 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Apr 19 16:18:02 2016 -0700
----------------------------------------------------------------------
precommit/personality/flink.sh | 61 +++++++------
precommit/personality/hbase.sh | 166 +++++++++++++++++++++++++++++++----
precommit/personality/jmeter.sh | 3 +
3 files changed, 180 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/64377ca5/precommit/personality/flink.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/flink.sh b/precommit/personality/flink.sh
index 6b4f0a8..76fa418 100755
--- a/precommit/personality/flink.sh
+++ b/precommit/personality/flink.sh
@@ -31,6 +31,10 @@ function personality_globals
add_test_type flinklib
+## @description flinklib file filter
+## @audience private
+## @stability evolving
+## @param filename
function flinklib_filefilter
{
local filename=$1
@@ -42,6 +46,9 @@ function flinklib_filefilter
fi
}
+## @description count files in the lib dir
+## @audience private
+## @stability evolving
function flinklib_count
{
find "${BASEDIR}" \
@@ -50,42 +57,46 @@ function flinklib_count
| wc -l
}
-function flinklib_preapply
+## @description check fliblib
+## @audience private
+## @stability evolving
+## @param repostatus
+function flinklib_rebuild
{
+ declare repostatus=$1
+
start_clock
big_console_header "${PATCH_BRANCH} flink library dependencies"
if ! verify_needed_test flinklib; then
- echo "Patch does not need flinklib testing."
+ echo "${BUILDMODEMSG} does not need flinklib testing."
return 0
fi
pushd "${BASEDIR}" >/dev/null
- echo_and_redirect "${PATCH_DIR}/branch-flinklib-root.txt" \
+ echo_and_redirect "${PATCH_DIR}/${repostatus}-flinklib-root.txt" \
"${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
if [[ $? != 0 ]]; then
add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}."
fi
- FLINK_PRE_LIB_FILES=$(flinklib_count)
- popd >/dev/null
-}
-function flinklib_postapply
-{
- start_clock
- big_console_header "Patch flink library dependencies"
-
- if ! verify_needed_test flinklib; then
- echo "Patch does not need flinklib testing."
+ if [[ ${repostatus} = branch ]]; then
+ FLINK_PRE_LIB_FILES=$(flinklib_count)
return 0
+ else
+ FLINK_POST_LIB_FILES=$(flinklib_count)
fi
-
- pushd "${BASEDIR}" >/dev/null
- echo_and_redirect "${PATCH_DIR}/patch-flinklib-root.txt" \
- "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
- FLINK_POST_LIB_FILES=$(flinklib_count)
popd >/dev/null
+ if [[ "${BUILDMODE}" = full ]]; then
+ if [[ ${FLINK_POST_LIB_FILES} -gt 0 ]]; then
+ add_vote_table -1 flinklib "Lib folder dependencies are currently ${FLINK_POST_LIB_FILES}"
+ return 1
+ else
+ add_vote_table +1 flinklib "No lib folder dependencies!"
+ return 0
+ fi
+ fi
if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then
add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \
@@ -99,16 +110,4 @@ function flinklib_postapply
"$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))."
fi
return 0
-}
-
-function flinklib_rebuild
-{
- declare repostatus=$1
-
- if [[ "${repostatus}" = branch ]]; then
- flinklib_preapply
- else
- flinklib_postinstall
- fi
-}
-
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/64377ca5/precommit/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/hbase.sh b/precommit/personality/hbase.sh
index e9e4ea8..5e6f0dc 100755
--- a/precommit/personality/hbase.sh
+++ b/precommit/personality/hbase.sh
@@ -22,17 +22,23 @@ personality_plugins "all"
function personality_globals
{
#shellcheck disable=SC2034
+ PROJECT_NAME=hbase
+ #shellcheck disable=SC2034
PATCH_BRANCH_DEFAULT=master
#shellcheck disable=SC2034
JIRA_ISSUE_RE='^HBASE-[0-9]+$'
#shellcheck disable=SC2034
GITHUB_REPO="apache/hbase"
+ # TODO use PATCH_BRANCH to select hadoop versions to use.
# All supported Hadoop versions that we want to test the compilation with
- HBASE_HADOOP_VERSIONS="2.4.1 2.5.2 2.6.0"
+ HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1"
+
+ # TODO use PATCH_BRANCH to select jdk versions to use.
# Override the maven options
MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
+
}
## @description Queue up modules for this personality
@@ -54,16 +60,19 @@ function personality_modules
if [[ ${repostatus} == branch
&& ${testtype} == mvninstall ]] ||
- [[ "${BUILDMODE}" == full ]]; then
- personality_enqueue_module . ${extra}
- return
- fi
+ [[ "${BUILDMODE}" == full ]];then
+ personality_enqueue_module . ${extra}
+ return
+ fi
if [[ ${testtype} = findbugs ]]; then
for module in "${CHANGED_MODULES[@]}"; do
- # skip findbugs on hbase-shell
+ # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
+ # in src/main/java where findbugs goes to look
if [[ ${module} == hbase-shell ]]; then
continue
+ elif [[ ${module} == hbase-it ]]; then
+ continue
else
# shellcheck disable=SC2086
personality_enqueue_module ${module} ${extra}
@@ -72,6 +81,16 @@ function personality_modules
return
fi
+ if [[ ${testtype} = unit ]]; then
+ extra="${extra} -PrunAllTests"
+
+ # Inject the jenkins build-id for our surefire invocations
+ # Used by zombie detection stuff, even though we're not including that yet.
+ if [ -n "${BUILD_ID}" ]; then
+ extra="${extra} -Dbuild.id=${BUILD_ID}"
+ fi
+ fi
+
for module in "${CHANGED_MODULES[@]}"; do
# shellcheck disable=SC2086
personality_enqueue_module ${module} ${extra}
@@ -79,9 +98,19 @@ function personality_modules
}
###################################################
+# Below here are our one-off tests specific to hbase.
+# TODO break them into individual files so it's easier to maintain them?
+
+# TODO line length check? could ignore all java files since checkstyle gets them.
+
+###################################################
add_test_type hadoopcheck
+## @description hadoopcheck file filter
+## @audience private
+## @stability evolving
+## @param filename
function hadoopcheck_filefilter
{
local filename=$1
@@ -91,6 +120,10 @@ function hadoopcheck_filefilter
fi
}
+## @description hadoopcheck test
+## @audience private
+## @stability evolving
+## @param repostatus
function hadoopcheck_rebuild
{
local repostatus=$1
@@ -114,7 +147,7 @@ function hadoopcheck_rebuild
-Dhadoop-two.version="${hadoopver}"
count=$(${GREP} -c ERROR "${logfile}")
if [[ ${count} -gt 0 ]]; then
- add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${hadoopver}."
+ add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
((result=result+1))
fi
done
@@ -123,14 +156,19 @@ function hadoopcheck_rebuild
return 1
fi
- add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
+ add_vote_table +1 hadoopcheck "${BUILDMODEMSG} does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
return 0
}
######################################
+# TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
add_test_type hbaseprotoc
+## @description hbaseprotoc file filter
+## @audience private
+## @stability evolving
+## @param filename
function hbaseprotoc_filefilter
{
local filename=$1
@@ -140,33 +178,38 @@ function hbaseprotoc_filefilter
fi
}
+## @description hadoopcheck test
+## @audience private
+## @stability evolving
+## @param repostatus
function hbaseprotoc_rebuild
{
- local i=0
- local fn
- local module
- local logfile
- local count
- local result
+ declare repostatus=$1
+ declare i=0
+ declare fn
+ declare module
+ declare logfile
+ declare count
+ declare result
if [[ "${repostatus}" = branch ]]; then
return 0
fi
- if ! verify_needed_test hbaseprotoc; then
+ verify_needed_test hbaseprotoc
+ if [[ $? == 0 ]]; then
return 0
fi
- big_console_header "Patch HBase protoc plugin"
+ big_console_header "HBase protoc plugin: ${BUILDMODE}"
start_clock
-
personality_modules patch hbaseprotoc
modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
# shellcheck disable=SC2153
- until [[ $i -eq ${#MODULE[@]} ]]; do
+ until [[ $i -eq "${#MODULE[@]}" ]]; do
if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
((result=result+1))
((i=i+1))
@@ -197,6 +240,10 @@ function hbaseprotoc_rebuild
add_test_type hbaseanti
+## @description hbaseanti file filter
+## @audience private
+## @stability evolving
+## @param filename
function hbaseanti_filefilter
{
local filename=$1
@@ -206,13 +253,22 @@ function hbaseanti_filefilter
fi
}
+## @description hbaseanti patch file check
+## @audience private
+## @stability evolving
+## @param filename
function hbaseanti_patchfile
{
local patchfile=$1
local warnings
local result
- if ! verify_needed_test hbaseanti; then
+ if [[ "${BUILDMODE}" = full ]]; then
+ return 0
+ fi
+
+ verify_needed_test hbaseanti
+ if [[ $? == 0 ]]; then
return 0
fi
@@ -239,3 +295,75 @@ function hbaseanti_patchfile
add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
return 0
}
+
+
+## @description hbase custom mvnsite file filter. See HBASE-15042
+## @audience private
+## @stability evolving
+## @param filename
+function mvnsite_filefilter
+{
+ local filename=$1
+
+ if [[ ${BUILDTOOL} = maven ]]; then
+ if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then
+ yetus_debug "tests/mvnsite: ${filename}"
+ add_test mvnsite
+ fi
+ fi
+}
+
+## This is named so that yetus will check us right after running tests.
+## Essentially, we check for normal failures and then we look for zombies.
+#function hbase_unit_logfilter
+#{
+# declare testtype="unit"
+# declare input=$1
+# declare output=$2
+# declare processes
+# declare process_output
+# declare zombies
+# declare zombie_count=0
+# declare zombie_process
+#
+# yetus_debug "in hbase-specific unit logfilter."
+#
+# # pass-through to whatever is counting actual failures
+# if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
+# "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
+# elif declare -f ${testtype}_logfilter >/dev/null; then
+# "${testtype}_logfilter" "${input}" "${output}"
+# fi
+#
+# start_clock
+# if [ -n "${BUILD_ID}" ]; then
+# yetus_debug "Checking for zombie test processes."
+# processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+# if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
+# yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
+# yetus_debug "${processes}"
+# sleep 30
+# #shellcheck disable=SC2016
+# for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
+# # Test our zombie still running (and that it still an hbase build item)
+# process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+# if [[ -n "${process_output}" ]]; then
+# yetus_error "Zombie: ${process_output}"
+# ((zombie_count = zombie_count + 1))
+# zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
+# zombies="${zombies} ${zombie_process}"
+# fi
+# done
+# fi
+# if [ "${zombie_count}" -ne 0 ]; then
+# add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
+# populate_test_table "zombie unit tests" "${zombies}"
+# else
+# yetus_info "Zombie check complete. All test runs exited normally."
+# stop_clock
+# fi
+# else
+# add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
+# fi
+#
+#}
http://git-wip-us.apache.org/repos/asf/yetus/blob/64377ca5/precommit/personality/jmeter.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/jmeter.sh b/precommit/personality/jmeter.sh
index 194fc4f..b4e61b5 100755
--- a/precommit/personality/jmeter.sh
+++ b/precommit/personality/jmeter.sh
@@ -58,6 +58,9 @@ function jmeter_parse_args
done
}
+## @description Download jmetere dependencies
+## @audience private
+## @stability evolving
function jmeter_precheck
{
if [[ ${JMETER_DOWNLOAD_JARS} = true ]]; then