You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2018/11/11 22:19:28 UTC
[12/17] yetus git commit: YETUS-15. build environment
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/patchfiles.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/core.d/patchfiles.sh b/precommit/src/main/shell/core.d/patchfiles.sh
new file mode 100755
index 0000000..90fb280
--- /dev/null
+++ b/precommit/src/main/shell/core.d/patchfiles.sh
@@ -0,0 +1,342 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## @description Use curl to download the patch as a last resort
+## @audience private
+## @stability evolving
+## @param patchloc
+## @param output
+## @return 0 got something
+## @return 1 error
+function generic_locate_patch
+{
+ declare input=$1
+ declare output=$2
+
+ if [[ "${OFFLINE}" == true ]]; then
+ yetus_debug "generic_locate_patch: offline, skipping"
+ return 1
+ fi
+
+ ${CURL} --silent -L \
+ --output "${output}" \
+ "${input}"
+ if [[ $? != 0 ]]; then
+ yetus_debug "generic_locate_patch: failed to download the patch."
+ return 1
+ fi
+ return 0
+}
+
+## @description Given a possible patch file, guess if it's a patch file
+## @description only using the more intense verify if we really need to
+## @audience private
+## @stability evolving
+## @param path to patch file to test
+## @return 0 we think it's a patch file
+## @return 1 we think it's not a patch file
+function guess_patch_file
+{
+ declare patch=$1
+ declare fileOutput
+
+ if [[ ! -f ${patch} ]]; then
+ return 1
+ fi
+
+ yetus_debug "Trying to guess if ${patch} is a patch file."
+ fileOutput=$("${FILE}" "${patch}")
+ if [[ $fileOutput =~ \ diff\ ]]; then
+ yetus_debug "file magic says it's a diff."
+ return 0
+ fi
+
+ fileOutput=$(head -n 1 "${patch}" | "${GREP}" -E "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
+ if [[ $? == 0 ]]; then
+ yetus_debug "first line looks like a patch file."
+ return 0
+ fi
+
+ patchfile_dryrun_driver "${patch}"
+}
+
+## @description Given ${PATCH_OR_ISSUE}, determine what type of patch file is in use,
+## @description and do the necessary work to place it into ${PATCH_DIR}/patch.
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure, may exit
+function locate_patch
+{
+ declare bugsys
+ declare patchfile=""
+ declare gotit=false
+
+ yetus_debug "locate patch"
+
+ if [[ -z "${PATCH_OR_ISSUE}" ]]; then
+ yetus_error "ERROR: No patch provided."
+ cleanup_and_exit 1
+ fi
+
+ echo "Processing: ${PATCH_OR_ISSUE}"
+ # it's a declarely provided file
+ if [[ -f ${PATCH_OR_ISSUE} ]]; then
+ patchfile="${PATCH_OR_ISSUE}"
+ PATCH_SYSTEM=generic
+ if [[ -f "${PATCH_DIR}/patch" ]]; then
+ "${DIFF}" -q "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" >/dev/null
+ if [[ $? -eq 1 ]]; then
+ rm "${PATCH_DIR}/patch"
+ fi
+ fi
+ else
+ # run through the bug systems. maybe they know?
+ for bugsys in ${BUGSYSTEMS}; do
+ if declare -f ${bugsys}_locate_patch >/dev/null 2>&1; then
+ "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
+ if [[ $? == 0 ]]; then
+ gotit=true
+ PATCH_SYSTEM=${bugsys}
+ fi
+ fi
+ done
+
+ # ok, none of the bug systems know. let's see how smart we are
+ if [[ ${gotit} == false ]]; then
+ generic_locate_patch "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}."
+ cleanup_and_exit 1
+ fi
+ PATCH_SYSTEM=generic
+ fi
+ fi
+
+ yetus_debug "Determined patch system to be ${PATCH_SYSTEM}"
+
+ if [[ ! -f "${PATCH_DIR}/patch"
+ && -f "${patchfile}" ]]; then
+ cp "${patchfile}" "${PATCH_DIR}/patch"
+ if [[ $? == 0 ]] ; then
+ echo "Patch file ${patchfile} copied to ${PATCH_DIR}"
+ else
+ yetus_error "ERROR: Could not copy ${patchfile} to ${PATCH_DIR}"
+ cleanup_and_exit 1
+ fi
+ fi
+}
+
+## @description if patch-level zero, then verify we aren't
+## @description just adding files
+## @audience public
+## @stability stable
+## @param log filename
+## @replaceable no
+## @return $?
+function patchfile_verify_zero
+{
+ declare logfile=$1
+ shift
+ declare dir
+ declare changed_files1
+ declare changed_files2
+ declare filename
+
+ # don't return /dev/null
+ # shellcheck disable=SC2016
+ changed_files1=$(${AWK} 'function p(s){if(s!~"^/dev/null"){print s}}
+ /^diff --git / { p($3); p($4) }
+ /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
+
+ # maybe we interpreted the patch wrong? check the log file
+ # shellcheck disable=SC2016
+ changed_files2=$(${GREP} -E '^[cC]heck' "${logfile}" \
+ | ${AWK} '{print $3}' \
+ | ${SED} -e 's,\.\.\.$,,g')
+
+ for filename in ${changed_files1} ${changed_files2}; do
+
+ # leading prefix = bad
+ if [[ ${filename} =~ ^(a|b)/ ]]; then
+ return 1
+ fi
+
+ # touching an existing file is proof enough
+ # that pl=0 is good
+ if [[ -f ${filename} ]]; then
+ return 0
+ fi
+
+ dir=$(dirname "${filename}" 2>/dev/null)
+ if [[ -n ${dir} && -d ${dir} ]]; then
+ return 0
+ fi
+ done
+
+ # ¯\_(ツ)_/¯ - no way for us to know, all new files with no prefix!
+ yetus_error "WARNING: Patch only adds files; using patch level ${PATCH_LEVEL}"
+ return 0
+}
+
+## @description git apply dryrun
+## @replaceable no
+## @audience private
+## @stability evolving
+## @param path to patch file to dryrun
+function gitapply_dryrun
+{
+ declare patchfile=$1
+ declare prefixsize=${2:-0}
+
+ while [[ ${prefixsize} -lt 4
+ && -z ${PATCH_METHOD} ]]; do
+ yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \
+ "${GIT}" apply --binary -v --check "-p${prefixsize}" "${patchfile}"
+ if [[ $? == 0 ]]; then
+ PATCH_LEVEL=${prefixsize}
+ PATCH_METHOD=gitapply
+ break
+ fi
+ ((prefixsize=prefixsize+1))
+ done
+
+ if [[ ${prefixsize} -eq 0 ]]; then
+ patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log"
+ if [[ $? != 0 ]]; then
+ PATCH_METHOD=""
+ PATCH_LEVEL=""
+ gitapply_dryrun "${patchfile}" 1
+ fi
+ fi
+}
+
+## @description patch patch dryrun
+## @replaceable no
+## @audience private
+## @stability evolving
+## @param path to patch file to dryrun
+function patchcmd_dryrun
+{
+ declare patchfile=$1
+ declare prefixsize=${2:-0}
+
+ while [[ ${prefixsize} -lt 4
+ && -z ${PATCH_METHOD} ]]; do
+ # shellcheck disable=SC2153
+ yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \
+ "${PATCH}" "-p${prefixsize}" -E --dry-run < "${patchfile}"
+ if [[ $? == 0 ]]; then
+ PATCH_LEVEL=${prefixsize}
+ PATCH_METHOD=patchcmd
+ break
+ fi
+ ((prefixsize=prefixsize+1))
+ done
+
+ if [[ ${prefixsize} -eq 0 ]]; then
+ patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log"
+ if [[ $? != 0 ]]; then
+ PATCH_METHOD=""
+ PATCH_LEVEL=""
+ patchcmd_dryrun "${patchfile}" 1
+ fi
+ fi
+}
+
+## @description driver for dryrun methods
+## @replaceable no
+## @audience private
+## @stability evolving
+## @param path to patch file to dryrun
+function patchfile_dryrun_driver
+{
+ declare patchfile=$1
+ declare method
+
+ #shellcheck disable=SC2153
+ for method in "${PATCH_METHODS[@]}"; do
+ if declare -f ${method}_dryrun >/dev/null; then
+ "${method}_dryrun" "${patchfile}"
+ fi
+ if [[ -n ${PATCH_METHOD} ]]; then
+ break
+ fi
+ done
+
+ if [[ -n ${PATCH_METHOD} ]]; then
+ return 0
+ fi
+ return 1
+}
+
+## @description git patch apply
+## @replaceable no
+## @audience private
+## @stability evolving
+## @param path to patch file to apply
+function gitapply_apply
+{
+ declare patchfile=$1
+ declare extraopts
+
+ if [[ "${COMMITMODE}" = true ]]; then
+ extraopts="--whitespace=fix"
+ fi
+
+ echo "Applying the patch:"
+ yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-apply.log" \
+ "${GIT}" apply --binary ${extraopts} -v --stat --apply "-p${PATCH_LEVEL}" "${patchfile}"
+ ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-apply.log"
+}
+
+## @description patch patch apply
+## @replaceable no
+## @audience private
+## @stability evolving
+## @param path to patch file to apply
+function patchcmd_apply
+{
+ declare patchfile=$1
+
+ echo "Applying the patch:"
+ yetus_run_and_redirect "${PATCH_DIR}/apply-patch-patch-apply.log" \
+ "${PATCH}" "-p${PATCH_LEVEL}" -E < "${patchfile}"
+ cat "${PATCH_DIR}/apply-patch-patch-apply.log"
+}
+
+## @description driver for patch apply methods
+## @replaceable no
+## @audience private
+## @stability evolving
+## @param path to patch file to apply
+function patchfile_apply_driver
+{
+ declare patchfile=$1
+ declare gpg=$2
+
+ if declare -f ${PATCH_METHOD}_apply >/dev/null; then
+ "${PATCH_METHOD}_apply" "${patchfile}" "${gpg}"
+ if [[ $? -gt 0 ]]; then
+ return 1
+ fi
+ else
+ yetus_error "ERROR: Patching method ${PATCH_METHOD} does not have a way to apply patches!"
+ return 1
+ fi
+ return 0
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/core.d/reaper.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/core.d/reaper.sh b/precommit/src/main/shell/core.d/reaper.sh
new file mode 100755
index 0000000..ce48c25
--- /dev/null
+++ b/precommit/src/main/shell/core.d/reaper.sh
@@ -0,0 +1,252 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REAPER_MODE=off # off, report, kill
+declare -i REAPER_TOTAL_COUNT=0
+REAPER_DOCKER_ONLY=true
+REAPER_ZOMBIE_MODULES=()
+REAPER_ZOMBIE_LOGS=()
+declare -a REAPER_NAMES
+
+
+## @description Add a regex to the reaper's checklist
+## @description NOTE: Users WILL override anything added before
+## @description argument parsing!
+## @stability evolving
+## @audience public
+## @replaceable no
+function reaper_add_name
+{
+ yetus_add_array_element REAPER_NAMES "$1"
+}
+
+## @description Reaper-specific usage
+## @stability stable
+## @audience private
+## @replaceable no
+function reaper_usage
+{
+ yetus_add_option "--reapermode={off,report,kill}" "Set unit test reaper mode (default: '${REAPER_MODE}')"
+ yetus_add_option "--reaperdockeronly=<bool>" "Only run the reaper in --docker (default: ${REAPER_DOCKER_ONLY})"
+ yetus_add_option "--reapernames=<list>" "List of regexs to search (default build tool dependent)"
+}
+
+## @description Reaper-specific argument parsing
+## @stability stable
+## @audience private
+## @replaceable no
+## @param arguments
+function reaper_parse_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --reapermode=*)
+ REAPER_MODE=${i#*=}
+ ;;
+ --reaperdockeronly=*)
+ REAPER_DOCKER_ONLY=${i#*=}
+ ;;
+ --reapernames=*)
+ yetus_comma_to_array REAPER_NAMES "${i#*=}"
+ ;;
+ esac
+ done
+
+ # Running the reaper outside of Dockermode is very dangerous
+
+ if [[ "${REAPER_DOCKER_ONLY}" = "true" && ${DOCKERMODE} = "false" ]]; then
+ REAPER_MODE="off"
+ return
+ fi
+
+ # make sure REAPER_MODE is something valid and turn us on
+ # as necessary
+ if [[ "${REAPER_MODE}" = "report" || "${REAPER_MODE}" = "kill" ]]; then
+ add_test_format reaper
+ yetus_add_entry EXEC_MODES Reaper
+ else
+ REAPER_MODE="off"
+ fi
+
+}
+
+## @description Initialize the reaper
+## @stability stable
+## @audience private
+## @replaceable yes
+## @param arguments
+function reaper_initialize
+{
+ determine_user
+}
+
+
+
+## @description Reaper coprocessor function that
+## @description runs outside the law
+## @stability evolving
+## @audience private
+## @replaceable yes
+function reaper_coproc_func
+{
+ declare line
+ declare i
+ declare module
+ declare filefrag
+ declare cmd
+ declare args
+ declare pid
+ declare -a pidlist
+ declare -i count
+
+ echo "Reaper watching for: ${REAPER_NAMES[*]}" >> "${PATCH_DIR}/reaper.txt"
+
+ while true; do
+ read -r cmd
+ case ${cmd} in
+ reap)
+
+ read -r module
+ read -r logfile
+
+ while read -r line; do
+ ((count=count+1))
+ for i in "${REAPER_NAMES[@]}"; do
+ echo "${line}" | ${GREP} -E "${i}" >> "${PATCH_DIR}/${logfile}"
+ done
+ done < <(ps -u "${USER_ID}" -o pid= -o args=)
+
+ pidlist=()
+ count=0
+ while read -r line; do
+ ((count=count+1))
+ pid=$(echo "${line}" | cut -f1 -d' ')
+ args=$(echo "${line}" | cut -f2- -d' ')
+ if [[ "${REAPER_MODE}" = "kill" ]]; then
+ pidlist+=("${pid}")
+ echo "Killing ${pid} ${args}" >> "${PATCH_DIR}/reaper.txt" 2>&1
+ fi
+ done < <(cat "${PATCH_DIR}/${logfile}")
+
+ # tell our parent how many
+ # doing this now means killing in the background
+ echo ${count}
+
+ if [[ ${count} -eq 0 ]]; then
+ rm "${PATCH_DIR}/${logfile}"
+ fi
+
+ for i in "${pidlist[@]}"; do
+ if [[ "${REAPER_MODE}" = "kill" ]]; then
+ pid_kill "${i}" >> "${PATCH_DIR}/reaper.txt" 2>&1
+ fi
+ done
+ ;;
+ exit)
+ exit 0
+ ;;
+ esac
+ done
+}
+
+## @description Run the reaper
+## @stability evolving
+## @audience private
+## @replaceable yes
+## @param module
+## @param testlog
+## @param testfrag
+function reaper_post_exec
+{
+ declare module=$1
+ declare filefrag=$2
+ declare count
+ declare myfile="${filefrag}-reaper.txt"
+ declare killmsg=""
+
+ case "${REAPER_MODE}" in
+ off)
+ return 0
+ ;;
+ kill)
+ killmsg=" and killed"
+ ;;
+ esac
+
+ yetus_debug "Checking for unreaped processes:"
+
+ # give some time for things to die naturally
+ sleep 2
+
+ #shellcheck disable=SC2154,SC2086
+ printf "reap\n%s\n%s\n" "${module}" "${myfile}" >&${reaper_coproc[1]}
+
+ #shellcheck disable=SC2154,SC2086
+ read -r count <&${reaper_coproc[0]}
+
+ if [[ ${count} -gt 0 ]]; then
+ ((REAPER_TOTAL_COUNT=REAPER_TOTAL_COUNT+count))
+ printf "\nFound%s %s left over processes\n\n" "${killmsg}" "${count}"
+ REAPER_ZOMBIE_MODULES+=("${module}:${count}")
+ REAPER_ZOMBIE_LOGS+=("@@BASE@@/${myfile}")
+ return 1
+ fi
+
+ return 0
+}
+
+## @description Reaper output to the user
+## @stability evolving
+## @audience private
+## @replaceable yes
+## @param jdkname
+function reaper_finalize_results
+{
+ declare jdk=$1
+ declare fn
+
+ if [[ "${REAPER_MODE}" = "off" ]]; then
+ return 0
+ fi
+
+ if [[ ${#REAPER_ZOMBIE_MODULES[@]} -gt 0 ]] ; then
+ populate_test_table "${jdk}Unreaped Processes" "${REAPER_ZOMBIE_MODULES[@]}"
+ for fn in "${REAPER_ZOMBIE_LOGS[@]}"; do
+ add_footer_table "Unreaped Processes Log" "${fn}"
+ done
+ REAPER_ZOMBIE_MODULES=()
+ REAPER_ZOMBIE_LOGS=()
+ fi
+}
+
+## @description Reaper output to the user
+## @stability evolving
+## @audience private
+## @replaceable yes
+## @param jdkname
+function reaper_total_count
+{
+
+ if [[ "${REAPER_MODE}" = "off" ]]; then
+ return 0
+ fi
+
+ if [[ ${REAPER_TOTAL_COUNT} -gt 0 ]]; then
+ add_vote_table -0 reaper "Unreaped process count: ${REAPER_TOTAL_COUNT}"
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/docker-cleanup.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/docker-cleanup.sh b/precommit/src/main/shell/docker-cleanup.sh
new file mode 100755
index 0000000..1d17b1b
--- /dev/null
+++ b/precommit/src/main/shell/docker-cleanup.sh
@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no shelldocs required from this file
+# SHELLDOC-IGNORE
+
+# Make sure that bash version meets the pre-requisite
+
+if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+ || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+ echo "bash v3.2+ is required. Sorry."
+ exit 1
+fi
+
+this="${BASH_SOURCE-$0}"
+BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+BINNAME=${this##*/}
+BINNAME=${BINNAME%.sh}
+#shellcheck disable=SC2034
+STARTINGDIR=$(pwd)
+#shellcheck disable=SC2034
+USER_PARAMS=("$@")
+#shellcheck disable=SC2034
+QATESTMODE=false
+
+# dummy functions
+function add_vote_table
+{
+ true
+}
+
+function add_footer_table
+{
+ true
+}
+
+function bugsystem_finalreport
+{
+ true
+}
+
+## @description import core library routines
+## @audience private
+## @stability evolving
+function import_core
+{
+ declare filename
+
+ for filename in "${BINDIR}/core.d"/*; do
+ # shellcheck disable=SC1091
+ # shellcheck source=core.d/01-common.sh
+ . "${filename}"
+ done
+}
+
+## @description import plugins then remove the stuff we don't need
+## @audience public
+## @stability stable
+## @replaceable no
+function import_and_clean
+{
+ importplugins
+ yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list"
+ unset BUILDTOOLS
+ unset TESTTYPES
+ unset TESTFORMATS
+
+ #shellcheck disable=SC2034
+ DOCKER_CLEANUP_CMD=true
+ #shellcheck disable=SC2034
+ DOCKERSUPPORT=true
+ #shellcheck disable=SC2034
+ ROBOT=true
+ #shellcheck disable=SC2034
+ DOCKERFAIL="fail"
+}
+
+## @description Clean the filesystem as appropriate and then exit
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param runresult
+function cleanup_and_exit
+{
+ local result=$1
+
+ if [[ ${PATCH_DIR} =~ ^/tmp/yetus
+ && -d ${PATCH_DIR} ]]; then
+ rm -rf "${PATCH_DIR}"
+ fi
+
+ # shellcheck disable=SC2086
+ exit ${result}
+}
+
+## @description Setup the default global variables
+## @audience public
+## @stability stable
+## @replaceable no
+function setup_defaults
+{
+ common_defaults
+}
+
+## @description Interpret the command line parameters
+## @audience private
+## @stability stable
+## @replaceable no
+## @param $@
+## @return May exit on failure
+function parse_args
+{
+ declare i
+
+ common_args "$@"
+
+ for i in "$@"; do
+ case ${i} in
+ --robot)
+ # shellcheck disable=SC2034
+ ROBOT=true
+ ;;
+ --sentinel)
+ # shellcheck disable=SC2034
+ ROBOT=true
+ # shellcheck disable=SC2034
+ SENTINEL=true
+ yetus_add_entry EXEC_MODES Sentinel
+ ;;
+ esac
+ done
+
+ docker_parse_args "$@"
+}
+
+## @description Print the usage information
+## @audience public
+## @stability stable
+## @replaceable no
+function yetus_usage
+{
+ import_and_clean
+
+ echo "${BINNAME} [OPTIONS]"
+
+ yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
+ yetus_add_option "--robot" "Assume this is an automated run"
+ yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)"
+ docker_usage
+
+ yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
+ yetus_reset_usage
+}
+
+## @description Large display for the user console
+## @audience public
+## @stability stable
+## @replaceable no
+## @param string
+## @return large chunk of text
+function big_console_header
+{
+ local text="$*"
+ local spacing=$(( (75+${#text}) /2 ))
+ printf "\n\n"
+ echo "============================================================================"
+ echo "============================================================================"
+ printf "%*s\n" ${spacing} "${text}"
+ echo "============================================================================"
+ echo "============================================================================"
+ printf "\n\n"
+}
+
+trap "cleanup_and_exit 1" HUP INT QUIT TERM
+
+import_core
+
+setup_defaults
+
+parse_args "$@"
+
+import_and_clean
+
+parse_args_plugins "$@"
+
+docker_initialize
+plugins_initialize
+
+docker_cleanup
+RESULT=$?
+
+cleanup_and_exit ${RESULT}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/it/it-tp.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/it/it-tp.sh b/precommit/src/main/shell/it/it-tp.sh
new file mode 100755
index 0000000..0e275f4
--- /dev/null
+++ b/precommit/src/main/shell/it/it-tp.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# where to get Apache Yetus
+YETUSDIR=${YETUSDIR:-$(pwd)/yetus}
+
+# project to work with
+PROJECT=${PROJECT:-hadoop}
+
+# temp directory to play in
+WORKDIR=${WORKDIR:-/tmp/yetus.${PROJECT}}
+
+# where to store artifacts: logs, etc. RELATIVE PATH!
+PATCHDIR=out
+
+# set the real dir where our source is. this is a relative path!
+BASEDIR=srcdir
+
+# dockerfile to use
+DOCKERFILE=${DOCKERFILE:-${BASEDIR}/dev-support/docker/Dockerfile}
+
+# make the directory if it doesn't exist
+mkdir -p "${WORKDIR}"
+
+# build out workdir
+pushd "${WORKDIR}" || exit 1
+
+# checkout the project's source
+if [[ ! -d "${BASEDIR}" ]]; then
+ git clone "https://github.com/apache/${PROJECT}" "${BASEDIR}"
+fi
+
+# clean out the out dir
+rm -rf "${PATCHDIR}" || true
+
+# make sure they exist
+mkdir -p "${PATCHDIR}"
+
+# if we abort the run in the middle of git, it will leave a present we
+# don't want
+if [[ -f "${BASEDIR}/.git/index.lock" ]]; then
+ rm "${BASEDIR}/.git/index.lock"
+fi
+
+# our 'default' args, in (mostly) alphabetical order
+
+# rsync these files back into the archive dir
+YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+
+# where the source is located
+YETUS_ARGS+=("--basedir=${BASEDIR}")
+
+# want to make sure the output is sane for these
+YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
+YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
+YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
+
+# run in docker mode
+YETUS_ARGS+=("--docker")
+
+# which Dockerfile to use
+YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+
+# force JDK to be OpenJDK 8
+YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
+
+# temp storage, etc
+YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
+
+# plugins to enable. modify as necessary based upon what is being tested
+YETUS_ARGS+=("--plugins=jira,maven,briefreport,htmlout")
+
+# Many projects need a high process limit
+YETUS_ARGS+=("--proclimit=5000")
+
+# project name. this will auto trigger personality for built-ins
+YETUS_ARGS+=("--project=${PROJECT}")
+
+# nuke the src repo before working
+YETUS_ARGS+=("--resetrepo")
+
+# run test-patch from the source tree specified up above
+TESTPATCHBIN=${YETUSDIR}/precommit/test-patch.sh
+
+# now run test-patch with any optional arguments:
+# --empty-patch for a full run aka 'qbt'
+# URL for a remote patch file
+# file name for local patch file
+# JIRA Issue, etc, etc.
+#
+# also, can add parameters or override the above as necessary
+
+cat <<EOF
+
+*******************
+Starting test-patch
+*******************
+
+EOF
+
+/bin/bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${@}"
+
+cat <<EOF
+
+*******************
+Stopping test-patch
+*******************
+
+EOF
+
+popd || exit 1
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/accumulo.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/accumulo.sh b/precommit/src/main/shell/personality/accumulo.sh
new file mode 100755
index 0000000..f2d3e7c
--- /dev/null
+++ b/precommit/src/main/shell/personality/accumulo.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run all plugins
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^ACCUMULO-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/accumulo"
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="http://accumulo.apache.org/git.html#contributors"
+ # We want to invoke the 'check' goal, not the default 'checkstyle'. We define
+ # our own checkstyle rules which isn't possible via 'checkstyle' configuration
+ #shellcheck disable=SC2034
+ CHECKSTYLE_GOAL="check"
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+ clear_personality_queue
+
+ if [[ ${testtype} == 'unit' ]]; then
+ # Run all tests, not just the tests in the modules affected
+ yetus_debug "Overriding to run all unit tests"
+
+ personality_enqueue_module .
+ return
+ fi
+
+ # Make sure we re-add the changed modules if we didn't short-circuit out
+ for module in "${CHANGED_MODULES[@]}"; do
+ personality_enqueue_module "${module}"
+ done
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/bigtop.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/bigtop.sh b/precommit/src/main/shell/personality/bigtop.sh
new file mode 100755
index 0000000..c374289
--- /dev/null
+++ b/precommit/src/main/shell/personality/bigtop.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all,-checkstyle,-findbugs"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ # shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^BIGTOP-[0-9]+$'
+ # shellcheck disable=SC2034
+ BUILDTOOL=gradle
+ # shellcheck disable=SC2034
+ GITHUB_REPO="apache/bigtop"
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/drill.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/drill.sh b/precommit/src/main/shell/personality/drill.sh
new file mode 100755
index 0000000..cc35d45
--- /dev/null
+++ b/precommit/src/main/shell/personality/drill.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/drill"
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="https://drill.apache.org/docs/apache-drill-contribution-guidelines/"
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^DRILL-[0-9]+$'
+ #shellcheck disable=SC2034
+ JIRA_STATUS_RE='Reviewable'
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/flink.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/flink.sh b/precommit/src/main/shell/personality/flink.sh
new file mode 100755
index 0000000..a9c0fdf
--- /dev/null
+++ b/precommit/src/main/shell/personality/flink.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^FLINK-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/flink"
+}
+
+add_test_type flinklib
+
+## @description flinklib file filter
+## @audience private
+## @stability evolving
+## @param filename
+function flinklib_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$
+ || ${filename} =~ \.scala$
+ || ${filename} =~ pom.xml$ ]]; then
+ add_test flinklib
+ fi
+}
+
+## @description count files in the lib dir
+## @audience private
+## @stability evolving
+function flinklib_count
+{
+ find "${BASEDIR}" \
+ | ${GREP} "/lib/" \
+ | ${GREP} -v "_qa_workdir" \
+ | wc -l
+}
+
+## @description check fliblib
+## @audience private
+## @stability evolving
+## @param repostatus
+function flinklib_rebuild
+{
+ declare repostatus=$1
+
+ start_clock
+ big_console_header "${PATCH_BRANCH} flink library dependencies"
+
+ if ! verify_needed_test flinklib; then
+ echo "${BUILDMODEMSG} does not need flinklib testing."
+ return 0
+ fi
+
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/${repostatus}-flinklib-root.txt" \
+ "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
+ if [[ $? != 0 ]]; then
+ add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}."
+ fi
+
+ if [[ ${repostatus} = branch ]]; then
+ FLINK_PRE_LIB_FILES=$(flinklib_count)
+ return 0
+ else
+ FLINK_POST_LIB_FILES=$(flinklib_count)
+ fi
+ popd >/dev/null
+
+ if [[ "${BUILDMODE}" = full ]]; then
+ if [[ ${FLINK_POST_LIB_FILES} -gt 0 ]]; then
+ add_vote_table -1 flinklib "Lib folder dependencies are currently ${FLINK_POST_LIB_FILES}"
+ return 1
+ else
+ add_vote_table +1 flinklib "No lib folder dependencies!"
+ return 0
+ fi
+ fi
+
+ if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then
+ add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \
+ "${FLINK_PRE_LIB_FILES} to ${FLINK_POST_LIB_FILES}"
+ return 1
+ elif [[ "${FLINK_POST_LIB_FILES}" -eq "${FLINK_PRE_LIB_FILES}" ]]; then
+ add_vote_table 0 flinklib "Patch did not change lib dependencies" \
+ " (still ${FLINK_PRE_LIB_FILES})"
+ else
+ add_vote_table +1 flinklib "Patch decreases lib folder dependencies by " \
+ "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))."
+ fi
+ return 0
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/geode.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/geode.sh b/precommit/src/main/shell/personality/geode.sh
new file mode 100755
index 0000000..9f5c0ec
--- /dev/null
+++ b/precommit/src/main/shell/personality/geode.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all,-ant,-javac,-scalac,-scaladoc"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=develop
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/GEODE/How+to+Contribute"
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^(GEODE)-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/incubator-geode"
+ #shellcheck disable=SC2034
+ BUILDTOOL=gradle
+# PYLINT_OPTIONS="--indent-string=' '"
+
+# HADOOP_MODULES=""
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/hadoop.sh b/precommit/src/main/shell/personality/hadoop.sh
new file mode 100755
index 0000000..2ad4459
--- /dev/null
+++ b/precommit/src/main/shell/personality/hadoop.sh
@@ -0,0 +1,563 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# SHELLDOC-IGNORE
+#
+# Override these to match Apache Hadoop's requirements
+
+personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=trunk
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="https://wiki.apache.org/hadoop/HowToContribute"
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS|HDDS)-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/hadoop"
+ #shellcheck disable=SC2034
+ PYLINT_OPTIONS="--indent-string=' '"
+
+ HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)}
+ if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then
+ HADOOP_HOMEBREW_DIR=/usr/local
+ fi
+}
+
+function personality_parse_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --hadoop-isal-prefix=*)
+ ISAL_HOME=${i#*=}
+ ;;
+ --hadoop-openssl-prefix=*)
+ OPENSSL_HOME=${i#*=}
+ ;;
+ --hadoop-snappy-prefix=*)
+ SNAPPY_HOME=${i#*=}
+ ;;
+ esac
+ done
+}
+
+## @description Calculate the actual module ordering
+## @audience private
+## @stability evolving
+## @param ordering
+function hadoop_order
+{
+ declare ordering=$1
+ declare hadoopm
+
+ if [[ ${ordering} = normal ]]; then
+ hadoopm="${CHANGED_MODULES[*]}"
+ elif [[ ${ordering} = union ]]; then
+ hadoopm="${CHANGED_UNION_MODULES}"
+ elif [[ ${ordering} = mvnsrc ]]; then
+ hadoopm="${MAVEN_SRC_MODULES[*]}"
+ elif [[ ${ordering} = mvnsrctest ]]; then
+ hadoopm="${MAVEN_SRCTEST_MODULES[*]}"
+ else
+ hadoopm="${ordering}"
+ fi
+ echo "${hadoopm}"
+}
+
+## @description Determine if it is safe to run parallel tests
+## @audience private
+## @stability evolving
+## @param ordering
+function hadoop_test_parallel
+{
+ if [[ -f "${BASEDIR}/pom.xml" ]]; then
+ HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
+ | head -1 \
+ | "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
+ | cut -f1 -d- )
+ export HADOOP_VERSION
+ else
+ return 1
+ fi
+
+ hmajor=${HADOOP_VERSION%%\.*}
+ hmajorminor=${HADOOP_VERSION%\.*}
+ hminor=${hmajorminor##*\.}
+ # ... and just for reference
+ #hmicro=${HADOOP_VERSION##*\.}
+
+ # Apache Hadoop v2.8.0 was the first one to really
+ # get working parallel unit tests
+ if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then
+ return 1
+ fi
+
+ return 0
+}
+
+## @description Install extra modules for unit tests
+## @audience private
+## @stability evolving
+## @param ordering
+function hadoop_unittest_prereqs
+{
+ declare input=$1
+ declare mods
+ declare need_common=0
+ declare building_common=0
+ declare module
+ declare flags
+ declare fn
+
+ # prior to running unit tests, hdfs needs libhadoop.so built
+ # if we're building root, then this extra work is moot
+
+ #shellcheck disable=SC2086
+ mods=$(hadoop_order ${input})
+
+ for module in ${mods}; do
+ if [[ ${module} = hadoop-hdfs-project* ]]; then
+ need_common=1
+ elif [[ ${module} = hadoop-common-project/hadoop-common
+ || ${module} = hadoop-common-project ]]; then
+ building_common=1
+ elif [[ ${module} = . ]]; then
+ return
+ fi
+ done
+
+ # Windows builds *ALWAYS* need hadoop-common compiled
+ case ${OSTYPE} in
+ Windows_NT|CYGWIN*|MINGW*|MSYS*)
+ need_common=1
+ ;;
+ esac
+
+ if [[ ${need_common} -eq 1
+ && ${building_common} -eq 0 ]]; then
+ echo "unit test pre-reqs:"
+ module="hadoop-common-project/hadoop-common"
+ fn=$(module_file_fragment "${module}")
+ flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
+ pushd "${BASEDIR}/${module}" >/dev/null
+ # shellcheck disable=SC2086
+ echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \
+ "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
+ popd >/dev/null
+ fi
+}
+
+## @description Calculate the flags/settings for yarn-ui v2 build
+## @description based upon the OS
+## @audience private
+## @stability evolving
+function yarn_ui2_flag
+{
+
+ if [[ ${BUILD_NATIVE} != true ]]; then
+ return
+ fi
+
+ # Now it only tested on Linux/OSX, don't enable the profile on
+ # windows until it get verified
+ case ${OSTYPE} in
+ Linux)
+ # shellcheck disable=SC2086
+ echo -Pyarn-ui
+ ;;
+ Darwin)
+ echo -Pyarn-ui
+ ;;
+ *)
+ # Do nothing
+ ;;
+ esac
+}
+
+## @description Calculate the flags/settings for native code
+## @description based upon the OS
+## @audience private
+## @stability evolving
+function hadoop_native_flags
+{
+ if [[ ${BUILD_NATIVE} != true ]]; then
+ return
+ fi
+
+ declare -a args
+
+ # Based upon HADOOP-11937
+ #
+ # Some notes:
+ #
+ # - getting fuse to compile on anything but Linux
+ # is always tricky.
+ # - Darwin assumes homebrew is in use.
+ # - HADOOP-12027 required for bzip2 on OS X.
+ # - bzip2 is broken in lots of places
+ # (the shared library is considered experimental)
+ # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2
+ #
+
+ args=("-Drequire.test.libhadoop")
+
+ if [[ -d "${ISAL_HOME}/include" ]]; then
+ args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}")
+ fi
+
+ if [[ -d "${OPENSSL_HOME}/include" ]]; then
+ args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}")
+ elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then
+ args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/")
+ fi
+
+ if [[ -d "${SNAPPY_HOME}/include" ]]; then
+ args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}")
+ elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then
+ args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy")
+ fi
+
+ case ${OSTYPE} in
+ Linux)
+ # shellcheck disable=SC2086
+ echo \
+ -Pnative \
+ -Drequire.fuse \
+ -Drequire.openssl \
+ -Drequire.snappy \
+ -Drequire.valgrind \
+ "${args[@]}"
+ ;;
+ Darwin)
+ echo \
+ "${args[@]}" \
+ -Pnative \
+ -Drequire.snappy \
+ -Drequire.openssl
+ ;;
+ Windows_NT|CYGWIN*|MINGW*|MSYS*)
+ echo \
+ "${args[@]}" \
+ -Drequire.snappy -Drequire.openssl -Pnative-win
+ ;;
+ *)
+ echo \
+ "${args[@]}"
+ ;;
+ esac
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ declare repostatus=$1
+ declare testtype=$2
+ declare extra=""
+ declare ordering="normal"
+ declare needflags=false
+ declare foundbats=false
+ declare flags
+ declare fn
+ declare i
+ declare hadoopm
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ asflicense)
+ # this is very fast and provides the full path if we do it from
+ # the root of the source
+ personality_enqueue_module .
+ return
+ ;;
+ checkstyle)
+ ordering="union"
+ extra="-DskipTests"
+ ;;
+ compile)
+ ordering="union"
+ extra="-DskipTests"
+ needflags=true
+
+ # if something in common changed, we build the whole world
+ if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then
+ yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . "
+ ordering="."
+ fi
+ ;;
+ distclean)
+ ordering="."
+ extra="-DskipTests"
+ ;;
+ javadoc)
+ if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+
+ if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then
+ echo "javadoc pre-reqs:"
+ for i in hadoop-project \
+ hadoop-common-project/hadoop-annotations; do
+ fn=$(module_file_fragment "${i}")
+ pushd "${BASEDIR}/${i}" >/dev/null
+ echo "cd ${i}"
+ echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \
+ "${MAVEN}" "${MAVEN_ARGS[@]}" install
+ popd >/dev/null
+ done
+ fi
+ extra="-Pdocs -DskipTests"
+ ;;
+ mvneclipse)
+ if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+ ;;
+ mvninstall)
+ extra="-DskipTests"
+ if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then
+ ordering=.
+ fi
+ ;;
+ mvnsite)
+ if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+ ;;
+ unit)
+ if [[ "${BUILDMODE}" = full ]]; then
+ ordering=mvnsrc
+ elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
+ ordering=.
+ fi
+
+ if [[ ${TEST_PARALLEL} = "true" ]] ; then
+ if hadoop_test_parallel; then
+ extra="-Pparallel-tests"
+ if [[ -n ${TEST_THREADS:-} ]]; then
+ extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
+ fi
+ fi
+ fi
+ needflags=true
+ hadoop_unittest_prereqs "${ordering}"
+
+ if ! verify_needed_test javac; then
+ yetus_debug "hadoop: javac not requested"
+ if ! verify_needed_test native; then
+ yetus_debug "hadoop: native not requested"
+ yetus_debug "hadoop: adding -DskipTests to unit test"
+ extra="-DskipTests"
+ fi
+ fi
+
+ for i in "${CHANGED_FILES[@]}"; do
+ if [[ "${i}" =~ \.bats ]]; then
+ foundbats=true
+ fi
+ done
+
+ if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then
+ yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile"
+ extra="${extra} -P!shelltest"
+ else
+ extra="${extra} -Pshelltest"
+ fi
+ ;;
+ *)
+ extra="-DskipTests"
+ ;;
+ esac
+
+ if [[ ${needflags} = true ]]; then
+ flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
+ extra="${extra} ${flags}"
+ fi
+
+ extra="-Ptest-patch ${extra}"
+
+ for module in $(hadoop_order ${ordering}); do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
+
+## @description Add tests based upon personality needs
+## @audience private
+## @stability evolving
+## @param filename
+function personality_file_tests
+{
+ declare filename=$1
+
+ yetus_debug "Using Hadoop-specific personality_file_tests"
+
+ if [[ ${filename} =~ src/main/webapp ]]; then
+ yetus_debug "tests/webapp: ${filename}"
+ add_test shadedclient
+ elif [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ || ${filename} =~ src/scripts
+ || ${filename} =~ src/test/scripts
+ || ${filename} =~ src/main/bin
+ || ${filename} =~ shellprofile\.d
+ || ${filename} =~ src/main/conf
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ add_test mvnsite
+ add_test unit
+ elif [[ ${filename} =~ \.md$
+ || ${filename} =~ \.md\.vm$
+ || ${filename} =~ src/site
+ ]]; then
+ yetus_debug "tests/site: ${filename}"
+ add_test mvnsite
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test compile
+ add_test cc
+ add_test mvnsite
+ add_test javac
+ add_test unit
+ elif [[ ${filename} =~ build.xml$
+ || ${filename} =~ pom.xml$
+ || ${filename} =~ \.java$
+ || ${filename} =~ src/main
+ ]]; then
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test compile
+ add_test javac
+ add_test javadoc
+ add_test mvninstall
+ add_test mvnsite
+ add_test unit
+ add_test shadedclient
+ fi
+
+ # if we change anything in here, e.g. the test scripts
+ # then run the client artifact tests
+ if [[ ${filename} =~ hadoop-client-modules ]]; then
+ add_test shadedclient
+ fi
+
+ if [[ ${filename} =~ src/test ]]; then
+ yetus_debug "tests: src/test"
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
+
+## @description Image to print on success
+## @audience private
+## @stability evolving
+function hadoop_console_success
+{
+ printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
+ printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
+ printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
+ printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
+ printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
+ printf "ICB8bXwgICB8bXwgIAo"
+}
+
+###################################################
+# Hadoop project specific check of IT for shaded artifacts
+
+add_test_type shadedclient
+
+## @description check for test modules and add test/plugins as needed
+## @audience private
+## @stability evolving
+function shadedclient_initialize
+{
+ maven_add_install shadedclient
+}
+
+## @description build client facing shaded artifacts and test them
+## @audience private
+## @stability evolving
+## @param repostatus
+function shadedclient_rebuild
+{
+ declare repostatus=$1
+ declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt"
+ declare module
+ declare -a modules=()
+
+ if [[ ${OSTYPE} = Windows_NT ||
+ ${OSTYPE} =~ ^CYGWIN.* ||
+ ${OSTYPE} =~ ^MINGW32.* ||
+ ${OSTYPE} =~ ^MSYS.* ]]; then
+ echo "hadoop personality: building on windows, skipping check of client artifacts."
+ return 0
+ fi
+
+ yetus_debug "hadoop personality: seeing if we need the test of client artifacts."
+ for module in hadoop-client-modules/hadoop-client-check-invariants \
+ hadoop-client-modules/hadoop-client-check-test-invariants \
+ hadoop-client-modules/hadoop-client-integration-tests; do
+ if [ -d "${module}" ]; then
+ yetus_debug "hadoop personality: test module '${module}' is present."
+ modules=( ${modules[@]} -pl ${module} )
+ fi
+ done
+ if [ ${#modules[@]} -eq 0 ]; then
+ echo "hadoop personality: no test modules present, skipping check of client artifacts."
+ return 0
+ fi
+
+ big_console_header "Checking client artifacts on ${repostatus}"
+
+ echo_and_redirect "${logfile}" \
+ "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
+ "${modules[@]}" \
+ -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
+
+ count=$(${GREP} -c '\[ERROR\]' "${logfile}")
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts."
+ return 1
+ fi
+
+ add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts."
+ return 0
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/hbase.sh b/precommit/src/main/shell/personality/hbase.sh
new file mode 100755
index 0000000..b48691e
--- /dev/null
+++ b/precommit/src/main/shell/personality/hbase.sh
@@ -0,0 +1,368 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PROJECT_NAME=hbase
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^HBASE-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/hbase"
+
+ # TODO use PATCH_BRANCH to select hadoop versions to use.
+ # All supported Hadoop versions that we want to test the compilation with
+ HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1"
+
+ # TODO use PATCH_BRANCH to select jdk versions to use.
+
+ # Override the maven options
+ MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
+
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ extra="-DHBasePatchProcess"
+
+ if [[ ${repostatus} == branch
+ && ${testtype} == mvninstall ]] ||
+ [[ "${BUILDMODE}" == full ]];then
+ personality_enqueue_module . ${extra}
+ return
+ fi
+
+ if [[ ${testtype} = findbugs ]]; then
+ for module in "${CHANGED_MODULES[@]}"; do
+ # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
+ # in src/main/java where findbugs goes to look
+ if [[ ${module} == hbase-shell ]]; then
+ continue
+ elif [[ ${module} == hbase-it ]]; then
+ continue
+ else
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ fi
+ done
+ return
+ fi
+
+ if [[ ${testtype} = unit ]]; then
+ extra="${extra} -PrunAllTests"
+
+ # Inject the jenkins build-id for our surefire invocations
+ # Used by zombie detection stuff, even though we're not including that yet.
+ if [ -n "${BUILD_ID}" ]; then
+ extra="${extra} -Dbuild.id=${BUILD_ID}"
+ fi
+ fi
+
+ for module in "${CHANGED_MODULES[@]}"; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
+
+###################################################
+# Below here are our one-off tests specific to hbase.
+# TODO break them into individual files so it's easier to maintain them?
+
+# TODO line length check? could ignore all java files since checkstyle gets them.
+
+###################################################
+
+add_test_type hadoopcheck
+
+## @description hadoopcheck file filter
+## @audience private
+## @stability evolving
+## @param filename
+function hadoopcheck_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test hadoopcheck
+ fi
+}
+
+## @description hadoopcheck test
+## @audience private
+## @stability evolving
+## @param repostatus
+function hadoopcheck_rebuild
+{
+ local repostatus=$1
+ local hadoopver
+ local logfile
+ local count
+ local result=0
+
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
+
+ big_console_header "Compiling against various Hadoop versions"
+
+ export MAVEN_OPTS="${MAVEN_OPTS}"
+ for hadoopver in ${HBASE_HADOOP_VERSIONS}; do
+ logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
+ echo_and_redirect "${logfile}" \
+ "${MAVEN}" clean install \
+ -DskipTests -DHBasePatchProcess \
+ -Dhadoop-two.version="${hadoopver}"
+ count=$(${GREP} -c ERROR "${logfile}")
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
+ ((result=result+1))
+ fi
+ done
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ add_vote_table +1 hadoopcheck "${BUILDMODEMSG} does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
+ return 0
+}
+
+######################################
+
+# TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
+add_test_type hbaseprotoc
+
+## @description hbaseprotoc file filter
+## @audience private
+## @stability evolving
+## @param filename
+function hbaseprotoc_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.proto$ ]]; then
+ add_test hbaseprotoc
+ fi
+}
+
+## @description hadoopcheck test
+## @audience private
+## @stability evolving
+## @param repostatus
+function hbaseprotoc_rebuild
+{
+ declare repostatus=$1
+ declare i=0
+ declare fn
+ declare module
+ declare logfile
+ declare count
+ declare result
+
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
+
+ if ! verify_needed_test hbaseprotoc; then
+ return 0
+ fi
+
+ big_console_header "HBase protoc plugin: ${BUILDMODE}"
+
+ start_clock
+
+ personality_modules patch hbaseprotoc
+ modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
+
+ # shellcheck disable=SC2153
+ until [[ $i -eq "${#MODULE[@]}" ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+ module=${MODULE[$i]}
+ fn=$(module_file_fragment "${module}")
+ logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
+
+ count=$(${GREP} -c ERROR "${logfile}")
+
+ if [[ ${count} -gt 0 ]]; then
+ module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
+ "${count} new protoc errors in ${module}."
+ ((result=result+1))
+ fi
+ ((i=i+1))
+ done
+
+ modules_messages patch hbaseprotoc true
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+######################################
+
+add_test_type hbaseanti
+
+## @description hbaseanti file filter
+## @audience private
+## @stability evolving
+## @param filename
+function hbaseanti_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test hbaseanti
+ fi
+}
+
+## @description hbaseanti patch file check
+## @audience private
+## @stability evolving
+## @param filename
+function hbaseanti_patchfile
+{
+ local patchfile=$1
+ local warnings
+ local result
+
+ if [[ "${BUILDMODE}" = full ]]; then
+ return 0
+ fi
+
+ if ! verify_needed_test hbaseanti; then
+ return 0
+ fi
+
+ big_console_header "Checking for known anti-patterns"
+
+ start_clock
+
+ warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}")
+ if [[ ${warnings} -gt 0 ]]; then
+ add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}."
+ ((result=result+1))
+ fi
+
+ warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}")
+ if [[ ${warnings} -gt 0 ]]; then
+ add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}."
+ ((result=result+1))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
+ return 0
+}
+
+
+## @description hbase custom mvnsite file filter. See HBASE-15042
+## @audience private
+## @stability evolving
+## @param filename
+function mvnsite_filefilter
+{
+ local filename=$1
+
+ if [[ ${BUILDTOOL} = maven ]]; then
+ if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then
+ yetus_debug "tests/mvnsite: ${filename}"
+ add_test mvnsite
+ fi
+ fi
+}
+
+## This is named so that yetus will check us right after running tests.
+## Essentially, we check for normal failures and then we look for zombies.
+#function hbase_unit_logfilter
+#{
+# declare testtype="unit"
+# declare input=$1
+# declare output=$2
+# declare processes
+# declare process_output
+# declare zombies
+# declare zombie_count=0
+# declare zombie_process
+#
+# yetus_debug "in hbase-specific unit logfilter."
+#
+# # pass-through to whatever is counting actual failures
+# if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
+# "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
+# elif declare -f ${testtype}_logfilter >/dev/null; then
+# "${testtype}_logfilter" "${input}" "${output}"
+# fi
+#
+# start_clock
+# if [ -n "${BUILD_ID}" ]; then
+# yetus_debug "Checking for zombie test processes."
+# processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+# if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
+# yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
+# yetus_debug "${processes}"
+# sleep 30
+# #shellcheck disable=SC2016
+# for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
+# # Test our zombie still running (and that it still an hbase build item)
+# process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
+# if [[ -n "${process_output}" ]]; then
+# yetus_error "Zombie: ${process_output}"
+# ((zombie_count = zombie_count + 1))
+# zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
+# zombies="${zombies} ${zombie_process}"
+# fi
+# done
+# fi
+# if [ "${zombie_count}" -ne 0 ]; then
+# add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
+# populate_test_table "zombie unit tests" "${zombies}"
+# else
+# yetus_info "Zombie check complete. All test runs exited normally."
+# stop_clock
+# fi
+# else
+# add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
+# fi
+#
+#}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/jmeter.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/jmeter.sh b/precommit/src/main/shell/personality/jmeter.sh
new file mode 100755
index 0000000..b4e61b5
--- /dev/null
+++ b/precommit/src/main/shell/personality/jmeter.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all,-javadoc,-findbugs,-asflicense"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=trunk
+ # shellcheck disable=SC2034
+ BUILDTOOL=ant
+ # shellcheck disable=SC2034
+ GITHUB_REPO="apache/jmeter"
+ # shellcheck disable=SC2034
+ JMETER_DOWNLOAD_JARS=false
+}
+
+add_test_type jmeter
+
+## @description Personality usage options
+## @audience private
+## @stability evolving
+function jmeter_usage
+{
+ yetus_add_option "--jmeter-download-jars=<bool>" "download third-party jars needed by ant build"
+}
+
+## @description Process personality options
+## @audience private
+## @stability evolving
+## @param arguments
+function jmeter_parse_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --jmeter-download-jars=*)
+ JMETER_DOWNLOAD_JARS=${i#*=}
+ ;;
+ esac
+ done
+}
+
+## @description Download jmetere dependencies
+## @audience private
+## @stability evolving
+function jmeter_precheck
+{
+ if [[ ${JMETER_DOWNLOAD_JARS} = true ]]; then
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/jmeter-branch-download-jars.txt" "${ANT}" download_jars
+ popd >/dev/null
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/kafka.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/kafka.sh b/precommit/src/main/shell/personality/kafka.sh
new file mode 100755
index 0000000..0c2f749
--- /dev/null
+++ b/precommit/src/main/shell/personality/kafka.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all,-checkstyle,-asflicense"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=trunk
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^KAFKA-[0-9]+$'
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="http://kafka.apache.org/contributing.html"
+ # shellcheck disable=SC2034
+ BUILDTOOL=gradle
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/kafka"
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ declare repostatus=$1
+ declare testtype=$2
+ declare module
+ declare extra=""
+
+ yetus_debug "Using kafka personality_modules"
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ gradleboot)
+ # kafka's bootstrap is broken
+ if [[ ${testtype} == gradleboot ]]; then
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/kafka-configure-gradle.txt" gradle
+ popd >/dev/null
+ fi
+ ;;
+ compile)
+ extra="clean jar"
+ ;;
+ esac
+
+ for module in "${CHANGED_MODULES[@]}"; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module "${module}" ${extra}
+ done
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/orc.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/orc.sh b/precommit/src/main/shell/personality/orc.sh
new file mode 100755
index 0000000..0582b17
--- /dev/null
+++ b/precommit/src/main/shell/personality/orc.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all,-checkstyle,-findbugs"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^ORC-[0-9]+$'
+ # shellcheck disable=SC2034
+ BUILDTOOL=cmake
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/orc"
+
+ # protobuf fails
+ #shellcheck disable=SC2034
+ CMAKE_ROOT_BUILD=true
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/pig.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/pig.sh b/precommit/src/main/shell/personality/pig.sh
new file mode 100755
index 0000000..2014fda
--- /dev/null
+++ b/precommit/src/main/shell/personality/pig.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=trunk
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^PIG-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/pig"
+ #shellcheck disable=SC2034
+ BUILDTOOL=ant
+}
+
+## @description Queue up modules for this personality
+## @audience private
+## @stability evolving
+## @param repostatus
+## @param testtype
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ extra="-DPigPatchProcess= "
+
+ case ${testtype} in
+ findbugs)
+ # shellcheck disable=SC2034
+ ANT_FINDBUGSXML="${BASEDIR}/build/test/findbugs/pig-findbugs-report.xml"
+ extra="-Dfindbugs.home=${FINDBUGS_HOME}"
+ ;;
+ compile)
+ extra="${extra} -Djavac.args=-Xlint -Dcompile.c++=yes clean piggybank"
+ ;;
+ javadoc)
+ extra="${extra} -Dforrest.home=${FORREST_HOME}"
+ ;;
+ unit)
+ extra="${extra} -Dtest.junit.output.format=xml -Dcompile.c++=yes -Dtest.output=yes test-core"
+ ;;
+ esac
+
+ # shellcheck disable=SC2086
+ personality_enqueue_module . ${extra}
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/ratis.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/ratis.sh b/precommit/src/main/shell/personality/ratis.sh
new file mode 100755
index 0000000..d700512
--- /dev/null
+++ b/precommit/src/main/shell/personality/ratis.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^RATIS-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/incubator-ratis"
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/samza.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/samza.sh b/precommit/src/main/shell/personality/samza.sh
new file mode 100755
index 0000000..430ffbb
--- /dev/null
+++ b/precommit/src/main/shell/personality/samza.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all,-checkstyle"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^SAMZA-[0-9]+$'
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/SAMZA/Contributor's+Corner"
+ # shellcheck disable=SC2034
+ BUILDTOOL=gradle
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/samza"
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/tajo.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/tajo.sh b/precommit/src/main/shell/personality/tajo.sh
new file mode 100755
index 0000000..fb86ff7
--- /dev/null
+++ b/precommit/src/main/shell/personality/tajo.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^TAJO-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/tajo"
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/tez.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/tez.sh b/precommit/src/main/shell/personality/tez.sh
new file mode 100755
index 0000000..32d9ffc
--- /dev/null
+++ b/precommit/src/main/shell/personality/tez.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^TEZ-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/tez"
+ #shellcheck disable=SC2034
+ PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/src/main/shell/personality/yetus.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/personality/yetus.sh b/precommit/src/main/shell/personality/yetus.sh
new file mode 100755
index 0000000..3afa6d2
--- /dev/null
+++ b/precommit/src/main/shell/personality/yetus.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+personality_plugins "all"
+
+## @description Globals specific to this personality
+## @audience private
+## @stability evolving
+function personality_globals
+{
+ # shellcheck disable=SC2034
+ BUILDTOOL=maven
+ #shellcheck disable=SC2034
+ PATCH_BRANCH_DEFAULT=master
+ #shellcheck disable=SC2034
+ JIRA_ISSUE_RE='^YETUS-[0-9]+$'
+ #shellcheck disable=SC2034
+ GITHUB_REPO="apache/yetus"
+}