You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2018/11/11 22:19:30 UTC
[14/17] yetus git commit: YETUS-15. build environment
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/patchfiles.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/patchfiles.sh b/precommit/core.d/patchfiles.sh
deleted file mode 100755
index 90fb280..0000000
--- a/precommit/core.d/patchfiles.sh
+++ /dev/null
@@ -1,342 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description Use curl to download the patch as a last resort
-## @audience private
-## @stability evolving
-## @param patchloc
-## @param output
-## @return 0 got something
-## @return 1 error
-function generic_locate_patch
-{
- declare input=$1
- declare output=$2
-
- if [[ "${OFFLINE}" == true ]]; then
- yetus_debug "generic_locate_patch: offline, skipping"
- return 1
- fi
-
- ${CURL} --silent -L \
- --output "${output}" \
- "${input}"
- if [[ $? != 0 ]]; then
- yetus_debug "generic_locate_patch: failed to download the patch."
- return 1
- fi
- return 0
-}
-
-## @description Given a possible patch file, guess if it's a patch file
-## @description only using the more intense verify if we really need to
-## @audience private
-## @stability evolving
-## @param path to patch file to test
-## @return 0 we think it's a patch file
-## @return 1 we think it's not a patch file
-function guess_patch_file
-{
- declare patch=$1
- declare fileOutput
-
- if [[ ! -f ${patch} ]]; then
- return 1
- fi
-
- yetus_debug "Trying to guess if ${patch} is a patch file."
- fileOutput=$("${FILE}" "${patch}")
- if [[ $fileOutput =~ \ diff\ ]]; then
- yetus_debug "file magic says it's a diff."
- return 0
- fi
-
- fileOutput=$(head -n 1 "${patch}" | "${GREP}" -E "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
- if [[ $? == 0 ]]; then
- yetus_debug "first line looks like a patch file."
- return 0
- fi
-
- patchfile_dryrun_driver "${patch}"
-}
-
-## @description Given ${PATCH_OR_ISSUE}, determine what type of patch file is in use,
-## @description and do the necessary work to place it into ${PATCH_DIR}/patch.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure, may exit
-function locate_patch
-{
- declare bugsys
- declare patchfile=""
- declare gotit=false
-
- yetus_debug "locate patch"
-
- if [[ -z "${PATCH_OR_ISSUE}" ]]; then
- yetus_error "ERROR: No patch provided."
- cleanup_and_exit 1
- fi
-
- echo "Processing: ${PATCH_OR_ISSUE}"
- # it's a declarely provided file
- if [[ -f ${PATCH_OR_ISSUE} ]]; then
- patchfile="${PATCH_OR_ISSUE}"
- PATCH_SYSTEM=generic
- if [[ -f "${PATCH_DIR}/patch" ]]; then
- "${DIFF}" -q "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch" >/dev/null
- if [[ $? -eq 1 ]]; then
- rm "${PATCH_DIR}/patch"
- fi
- fi
- else
- # run through the bug systems. maybe they know?
- for bugsys in ${BUGSYSTEMS}; do
- if declare -f ${bugsys}_locate_patch >/dev/null 2>&1; then
- "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
- if [[ $? == 0 ]]; then
- gotit=true
- PATCH_SYSTEM=${bugsys}
- fi
- fi
- done
-
- # ok, none of the bug systems know. let's see how smart we are
- if [[ ${gotit} == false ]]; then
- generic_locate_patch "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
- if [[ $? != 0 ]]; then
- yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}."
- cleanup_and_exit 1
- fi
- PATCH_SYSTEM=generic
- fi
- fi
-
- yetus_debug "Determined patch system to be ${PATCH_SYSTEM}"
-
- if [[ ! -f "${PATCH_DIR}/patch"
- && -f "${patchfile}" ]]; then
- cp "${patchfile}" "${PATCH_DIR}/patch"
- if [[ $? == 0 ]] ; then
- echo "Patch file ${patchfile} copied to ${PATCH_DIR}"
- else
- yetus_error "ERROR: Could not copy ${patchfile} to ${PATCH_DIR}"
- cleanup_and_exit 1
- fi
- fi
-}
-
-## @description if patch-level zero, then verify we aren't
-## @description just adding files
-## @audience public
-## @stability stable
-## @param log filename
-## @replaceable no
-## @return $?
-function patchfile_verify_zero
-{
- declare logfile=$1
- shift
- declare dir
- declare changed_files1
- declare changed_files2
- declare filename
-
- # don't return /dev/null
- # shellcheck disable=SC2016
- changed_files1=$(${AWK} 'function p(s){if(s!~"^/dev/null"){print s}}
- /^diff --git / { p($3); p($4) }
- /^(\+\+\+|---) / { p($2) }' "${PATCH_DIR}/patch" | sort -u)
-
- # maybe we interpreted the patch wrong? check the log file
- # shellcheck disable=SC2016
- changed_files2=$(${GREP} -E '^[cC]heck' "${logfile}" \
- | ${AWK} '{print $3}' \
- | ${SED} -e 's,\.\.\.$,,g')
-
- for filename in ${changed_files1} ${changed_files2}; do
-
- # leading prefix = bad
- if [[ ${filename} =~ ^(a|b)/ ]]; then
- return 1
- fi
-
- # touching an existing file is proof enough
- # that pl=0 is good
- if [[ -f ${filename} ]]; then
- return 0
- fi
-
- dir=$(dirname "${filename}" 2>/dev/null)
- if [[ -n ${dir} && -d ${dir} ]]; then
- return 0
- fi
- done
-
- # ¯\_(ツ)_/¯ - no way for us to know, all new files with no prefix!
- yetus_error "WARNING: Patch only adds files; using patch level ${PATCH_LEVEL}"
- return 0
-}
-
-## @description git apply dryrun
-## @replaceable no
-## @audience private
-## @stability evolving
-## @param path to patch file to dryrun
-function gitapply_dryrun
-{
- declare patchfile=$1
- declare prefixsize=${2:-0}
-
- while [[ ${prefixsize} -lt 4
- && -z ${PATCH_METHOD} ]]; do
- yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \
- "${GIT}" apply --binary -v --check "-p${prefixsize}" "${patchfile}"
- if [[ $? == 0 ]]; then
- PATCH_LEVEL=${prefixsize}
- PATCH_METHOD=gitapply
- break
- fi
- ((prefixsize=prefixsize+1))
- done
-
- if [[ ${prefixsize} -eq 0 ]]; then
- patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log"
- if [[ $? != 0 ]]; then
- PATCH_METHOD=""
- PATCH_LEVEL=""
- gitapply_dryrun "${patchfile}" 1
- fi
- fi
-}
-
-## @description patch patch dryrun
-## @replaceable no
-## @audience private
-## @stability evolving
-## @param path to patch file to dryrun
-function patchcmd_dryrun
-{
- declare patchfile=$1
- declare prefixsize=${2:-0}
-
- while [[ ${prefixsize} -lt 4
- && -z ${PATCH_METHOD} ]]; do
- # shellcheck disable=SC2153
- yetus_run_and_redirect "${PATCH_DIR}/patch-dryrun.log" \
- "${PATCH}" "-p${prefixsize}" -E --dry-run < "${patchfile}"
- if [[ $? == 0 ]]; then
- PATCH_LEVEL=${prefixsize}
- PATCH_METHOD=patchcmd
- break
- fi
- ((prefixsize=prefixsize+1))
- done
-
- if [[ ${prefixsize} -eq 0 ]]; then
- patchfile_verify_zero "${PATCH_DIR}/patch-dryrun.log"
- if [[ $? != 0 ]]; then
- PATCH_METHOD=""
- PATCH_LEVEL=""
- patchcmd_dryrun "${patchfile}" 1
- fi
- fi
-}
-
-## @description driver for dryrun methods
-## @replaceable no
-## @audience private
-## @stability evolving
-## @param path to patch file to dryrun
-function patchfile_dryrun_driver
-{
- declare patchfile=$1
- declare method
-
- #shellcheck disable=SC2153
- for method in "${PATCH_METHODS[@]}"; do
- if declare -f ${method}_dryrun >/dev/null; then
- "${method}_dryrun" "${patchfile}"
- fi
- if [[ -n ${PATCH_METHOD} ]]; then
- break
- fi
- done
-
- if [[ -n ${PATCH_METHOD} ]]; then
- return 0
- fi
- return 1
-}
-
-## @description git patch apply
-## @replaceable no
-## @audience private
-## @stability evolving
-## @param path to patch file to apply
-function gitapply_apply
-{
- declare patchfile=$1
- declare extraopts
-
- if [[ "${COMMITMODE}" = true ]]; then
- extraopts="--whitespace=fix"
- fi
-
- echo "Applying the patch:"
- yetus_run_and_redirect "${PATCH_DIR}/apply-patch-git-apply.log" \
- "${GIT}" apply --binary ${extraopts} -v --stat --apply "-p${PATCH_LEVEL}" "${patchfile}"
- ${GREP} -v "^Checking" "${PATCH_DIR}/apply-patch-git-apply.log"
-}
-
-## @description patch patch apply
-## @replaceable no
-## @audience private
-## @stability evolving
-## @param path to patch file to apply
-function patchcmd_apply
-{
- declare patchfile=$1
-
- echo "Applying the patch:"
- yetus_run_and_redirect "${PATCH_DIR}/apply-patch-patch-apply.log" \
- "${PATCH}" "-p${PATCH_LEVEL}" -E < "${patchfile}"
- cat "${PATCH_DIR}/apply-patch-patch-apply.log"
-}
-
-## @description driver for patch apply methods
-## @replaceable no
-## @audience private
-## @stability evolving
-## @param path to patch file to apply
-function patchfile_apply_driver
-{
- declare patchfile=$1
- declare gpg=$2
-
- if declare -f ${PATCH_METHOD}_apply >/dev/null; then
- "${PATCH_METHOD}_apply" "${patchfile}" "${gpg}"
- if [[ $? -gt 0 ]]; then
- return 1
- fi
- else
- yetus_error "ERROR: Patching method ${PATCH_METHOD} does not have a way to apply patches!"
- return 1
- fi
- return 0
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/reaper.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/reaper.sh b/precommit/core.d/reaper.sh
deleted file mode 100755
index ce48c25..0000000
--- a/precommit/core.d/reaper.sh
+++ /dev/null
@@ -1,252 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-REAPER_MODE=off # off, report, kill
-declare -i REAPER_TOTAL_COUNT=0
-REAPER_DOCKER_ONLY=true
-REAPER_ZOMBIE_MODULES=()
-REAPER_ZOMBIE_LOGS=()
-declare -a REAPER_NAMES
-
-
-## @description Add a regex to the reaper's checklist
-## @description NOTE: Users WILL override anything added before
-## @description argument parsing!
-## @stability evolving
-## @audience public
-## @replaceable no
-function reaper_add_name
-{
- yetus_add_array_element REAPER_NAMES "$1"
-}
-
-## @description Reaper-specific usage
-## @stability stable
-## @audience private
-## @replaceable no
-function reaper_usage
-{
- yetus_add_option "--reapermode={off,report,kill}" "Set unit test reaper mode (default: '${REAPER_MODE}')"
- yetus_add_option "--reaperdockeronly=<bool>" "Only run the reaper in --docker (default: ${REAPER_DOCKER_ONLY})"
- yetus_add_option "--reapernames=<list>" "List of regexs to search (default build tool dependent)"
-}
-
-## @description Reaper-specific argument parsing
-## @stability stable
-## @audience private
-## @replaceable no
-## @param arguments
-function reaper_parse_args
-{
- declare i
-
- for i in "$@"; do
- case ${i} in
- --reapermode=*)
- REAPER_MODE=${i#*=}
- ;;
- --reaperdockeronly=*)
- REAPER_DOCKER_ONLY=${i#*=}
- ;;
- --reapernames=*)
- yetus_comma_to_array REAPER_NAMES "${i#*=}"
- ;;
- esac
- done
-
- # Running the reaper outside of Dockermode is very dangerous
-
- if [[ "${REAPER_DOCKER_ONLY}" = "true" && ${DOCKERMODE} = "false" ]]; then
- REAPER_MODE="off"
- return
- fi
-
- # make sure REAPER_MODE is something valid and turn us on
- # as necessary
- if [[ "${REAPER_MODE}" = "report" || "${REAPER_MODE}" = "kill" ]]; then
- add_test_format reaper
- yetus_add_entry EXEC_MODES Reaper
- else
- REAPER_MODE="off"
- fi
-
-}
-
-## @description Initialize the reaper
-## @stability stable
-## @audience private
-## @replaceable yes
-## @param arguments
-function reaper_initialize
-{
- determine_user
-}
-
-
-
-## @description Reaper coprocessor function that
-## @description runs outside the law
-## @stability evolving
-## @audience private
-## @replaceable yes
-function reaper_coproc_func
-{
- declare line
- declare i
- declare module
- declare filefrag
- declare cmd
- declare args
- declare pid
- declare -a pidlist
- declare -i count
-
- echo "Reaper watching for: ${REAPER_NAMES[*]}" >> "${PATCH_DIR}/reaper.txt"
-
- while true; do
- read -r cmd
- case ${cmd} in
- reap)
-
- read -r module
- read -r logfile
-
- while read -r line; do
- ((count=count+1))
- for i in "${REAPER_NAMES[@]}"; do
- echo "${line}" | ${GREP} -E "${i}" >> "${PATCH_DIR}/${logfile}"
- done
- done < <(ps -u "${USER_ID}" -o pid= -o args=)
-
- pidlist=()
- count=0
- while read -r line; do
- ((count=count+1))
- pid=$(echo "${line}" | cut -f1 -d' ')
- args=$(echo "${line}" | cut -f2- -d' ')
- if [[ "${REAPER_MODE}" = "kill" ]]; then
- pidlist+=("${pid}")
- echo "Killing ${pid} ${args}" >> "${PATCH_DIR}/reaper.txt" 2>&1
- fi
- done < <(cat "${PATCH_DIR}/${logfile}")
-
- # tell our parent how many
- # doing this now means killing in the background
- echo ${count}
-
- if [[ ${count} -eq 0 ]]; then
- rm "${PATCH_DIR}/${logfile}"
- fi
-
- for i in "${pidlist[@]}"; do
- if [[ "${REAPER_MODE}" = "kill" ]]; then
- pid_kill "${i}" >> "${PATCH_DIR}/reaper.txt" 2>&1
- fi
- done
- ;;
- exit)
- exit 0
- ;;
- esac
- done
-}
-
-## @description Run the reaper
-## @stability evolving
-## @audience private
-## @replaceable yes
-## @param module
-## @param testlog
-## @param testfrag
-function reaper_post_exec
-{
- declare module=$1
- declare filefrag=$2
- declare count
- declare myfile="${filefrag}-reaper.txt"
- declare killmsg=""
-
- case "${REAPER_MODE}" in
- off)
- return 0
- ;;
- kill)
- killmsg=" and killed"
- ;;
- esac
-
- yetus_debug "Checking for unreaped processes:"
-
- # give some time for things to die naturally
- sleep 2
-
- #shellcheck disable=SC2154,SC2086
- printf "reap\n%s\n%s\n" "${module}" "${myfile}" >&${reaper_coproc[1]}
-
- #shellcheck disable=SC2154,SC2086
- read -r count <&${reaper_coproc[0]}
-
- if [[ ${count} -gt 0 ]]; then
- ((REAPER_TOTAL_COUNT=REAPER_TOTAL_COUNT+count))
- printf "\nFound%s %s left over processes\n\n" "${killmsg}" "${count}"
- REAPER_ZOMBIE_MODULES+=("${module}:${count}")
- REAPER_ZOMBIE_LOGS+=("@@BASE@@/${myfile}")
- return 1
- fi
-
- return 0
-}
-
-## @description Reaper output to the user
-## @stability evolving
-## @audience private
-## @replaceable yes
-## @param jdkname
-function reaper_finalize_results
-{
- declare jdk=$1
- declare fn
-
- if [[ "${REAPER_MODE}" = "off" ]]; then
- return 0
- fi
-
- if [[ ${#REAPER_ZOMBIE_MODULES[@]} -gt 0 ]] ; then
- populate_test_table "${jdk}Unreaped Processes" "${REAPER_ZOMBIE_MODULES[@]}"
- for fn in "${REAPER_ZOMBIE_LOGS[@]}"; do
- add_footer_table "Unreaped Processes Log" "${fn}"
- done
- REAPER_ZOMBIE_MODULES=()
- REAPER_ZOMBIE_LOGS=()
- fi
-}
-
-## @description Reaper output to the user
-## @stability evolving
-## @audience private
-## @replaceable yes
-## @param jdkname
-function reaper_total_count
-{
-
- if [[ "${REAPER_MODE}" = "off" ]]; then
- return 0
- fi
-
- if [[ ${REAPER_TOTAL_COUNT} -gt 0 ]]; then
- add_vote_table -0 reaper "Unreaped process count: ${REAPER_TOTAL_COUNT}"
- fi
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/docker-cleanup.sh
----------------------------------------------------------------------
diff --git a/precommit/docker-cleanup.sh b/precommit/docker-cleanup.sh
deleted file mode 100755
index 1d17b1b..0000000
--- a/precommit/docker-cleanup.sh
+++ /dev/null
@@ -1,206 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# no shelldocs required from this file
-# SHELLDOC-IGNORE
-
-# Make sure that bash version meets the pre-requisite
-
-if [[ -z "${BASH_VERSINFO[0]}" ]] \
- || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
- || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
- echo "bash v3.2+ is required. Sorry."
- exit 1
-fi
-
-this="${BASH_SOURCE-$0}"
-BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-BINNAME=${this##*/}
-BINNAME=${BINNAME%.sh}
-#shellcheck disable=SC2034
-STARTINGDIR=$(pwd)
-#shellcheck disable=SC2034
-USER_PARAMS=("$@")
-#shellcheck disable=SC2034
-QATESTMODE=false
-
-# dummy functions
-function add_vote_table
-{
- true
-}
-
-function add_footer_table
-{
- true
-}
-
-function bugsystem_finalreport
-{
- true
-}
-
-## @description import core library routines
-## @audience private
-## @stability evolving
-function import_core
-{
- declare filename
-
- for filename in "${BINDIR}/core.d"/*; do
- # shellcheck disable=SC1091
- # shellcheck source=core.d/01-common.sh
- . "${filename}"
- done
-}
-
-## @description import plugins then remove the stuff we don't need
-## @audience public
-## @stability stable
-## @replaceable no
-function import_and_clean
-{
- importplugins
- yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list"
- unset BUILDTOOLS
- unset TESTTYPES
- unset TESTFORMATS
-
- #shellcheck disable=SC2034
- DOCKER_CLEANUP_CMD=true
- #shellcheck disable=SC2034
- DOCKERSUPPORT=true
- #shellcheck disable=SC2034
- ROBOT=true
- #shellcheck disable=SC2034
- DOCKERFAIL="fail"
-}
-
-## @description Clean the filesystem as appropriate and then exit
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-function cleanup_and_exit
-{
- local result=$1
-
- if [[ ${PATCH_DIR} =~ ^/tmp/yetus
- && -d ${PATCH_DIR} ]]; then
- rm -rf "${PATCH_DIR}"
- fi
-
- # shellcheck disable=SC2086
- exit ${result}
-}
-
-## @description Setup the default global variables
-## @audience public
-## @stability stable
-## @replaceable no
-function setup_defaults
-{
- common_defaults
-}
-
-## @description Interpret the command line parameters
-## @audience private
-## @stability stable
-## @replaceable no
-## @param $@
-## @return May exit on failure
-function parse_args
-{
- declare i
-
- common_args "$@"
-
- for i in "$@"; do
- case ${i} in
- --robot)
- # shellcheck disable=SC2034
- ROBOT=true
- ;;
- --sentinel)
- # shellcheck disable=SC2034
- ROBOT=true
- # shellcheck disable=SC2034
- SENTINEL=true
- yetus_add_entry EXEC_MODES Sentinel
- ;;
- esac
- done
-
- docker_parse_args "$@"
-}
-
-## @description Print the usage information
-## @audience public
-## @stability stable
-## @replaceable no
-function yetus_usage
-{
- import_and_clean
-
- echo "${BINNAME} [OPTIONS]"
-
- yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
- yetus_add_option "--robot" "Assume this is an automated run"
- yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)"
- docker_usage
-
- yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
- yetus_reset_usage
-}
-
-## @description Large display for the user console
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-## @return large chunk of text
-function big_console_header
-{
- local text="$*"
- local spacing=$(( (75+${#text}) /2 ))
- printf "\n\n"
- echo "============================================================================"
- echo "============================================================================"
- printf "%*s\n" ${spacing} "${text}"
- echo "============================================================================"
- echo "============================================================================"
- printf "\n\n"
-}
-
-trap "cleanup_and_exit 1" HUP INT QUIT TERM
-
-import_core
-
-setup_defaults
-
-parse_args "$@"
-
-import_and_clean
-
-parse_args_plugins "$@"
-
-docker_initialize
-plugins_initialize
-
-docker_cleanup
-RESULT=$?
-
-cleanup_and_exit ${RESULT}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/it/it-tp.sh
----------------------------------------------------------------------
diff --git a/precommit/it/it-tp.sh b/precommit/it/it-tp.sh
deleted file mode 100755
index 0e275f4..0000000
--- a/precommit/it/it-tp.sh
+++ /dev/null
@@ -1,124 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# where to get Apache Yetus
-YETUSDIR=${YETUSDIR:-$(pwd)/yetus}
-
-# project to work with
-PROJECT=${PROJECT:-hadoop}
-
-# temp directory to play in
-WORKDIR=${WORKDIR:-/tmp/yetus.${PROJECT}}
-
-# where to store artifacts: logs, etc. RELATIVE PATH!
-PATCHDIR=out
-
-# set the real dir where our source is. this is a relative path!
-BASEDIR=srcdir
-
-# dockerfile to use
-DOCKERFILE=${DOCKERFILE:-${BASEDIR}/dev-support/docker/Dockerfile}
-
-# make the directory if it doesn't exist
-mkdir -p "${WORKDIR}"
-
-# build out workdir
-pushd "${WORKDIR}" || exit 1
-
-# checkout the project's source
-if [[ ! -d "${BASEDIR}" ]]; then
- git clone "https://github.com/apache/${PROJECT}" "${BASEDIR}"
-fi
-
-# clean out the out dir
-rm -rf "${PATCHDIR}" || true
-
-# make sure they exist
-mkdir -p "${PATCHDIR}"
-
-# if we abort the run in the middle of git, it will leave a present we
-# don't want
-if [[ -f "${BASEDIR}/.git/index.lock" ]]; then
- rm "${BASEDIR}/.git/index.lock"
-fi
-
-# our 'default' args, in (mostly) alphabetical order
-
-# rsync these files back into the archive dir
-YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
-
-# where the source is located
-YETUS_ARGS+=("--basedir=${BASEDIR}")
-
-# want to make sure the output is sane for these
-YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
-YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
-YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
-
-# run in docker mode
-YETUS_ARGS+=("--docker")
-
-# which Dockerfile to use
-YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
-
-# force JDK to be OpenJDK 8
-YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
-
-# temp storage, etc
-YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
-
-# plugins to enable. modify as necessary based upon what is being tested
-YETUS_ARGS+=("--plugins=jira,maven,briefreport,htmlout")
-
-# Many projects need a high process limit
-YETUS_ARGS+=("--proclimit=5000")
-
-# project name. this will auto trigger personality for built-ins
-YETUS_ARGS+=("--project=${PROJECT}")
-
-# nuke the src repo before working
-YETUS_ARGS+=("--resetrepo")
-
-# run test-patch from the source tree specified up above
-TESTPATCHBIN=${YETUSDIR}/precommit/test-patch.sh
-
-# now run test-patch with any optional arguments:
-# --empty-patch for a full run aka 'qbt'
-# URL for a remote patch file
-# file name for local patch file
-# JIRA Issue, etc, etc.
-#
-# also, can add parameters or override the above as necessary
-
-cat <<EOF
-
-*******************
-Starting test-patch
-*******************
-
-EOF
-
-/bin/bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${@}"
-
-cat <<EOF
-
-*******************
-Stopping test-patch
-*******************
-
-EOF
-
-popd || exit 1
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/jenkins/jenkins-admin.py
----------------------------------------------------------------------
diff --git a/precommit/jenkins/jenkins-admin.py b/precommit/jenkins/jenkins-admin.py
deleted file mode 100755
index f6ffdfe..0000000
--- a/precommit/jenkins/jenkins-admin.py
+++ /dev/null
@@ -1,245 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from optparse import OptionParser
-from tempfile import NamedTemporaryFile
-from xml.etree import ElementTree
-import base64
-import httplib
-import os
-import re
-import sys
-import urllib2
-
-def httpGet(resource, ignoreError=False, username=None, password=None):
- request = urllib2.Request(resource)
- if username and password:
- base64string = base64.b64encode('%s:%s' % (username, password))
- request.add_header("Authorization", "Basic %s" % base64string)
- try:
- response = urllib2.urlopen(request)
- except urllib2.HTTPError, http_err:
- code = http_err.code
- print '%s returns HTTP error %d: %s' \
- % (resource, code, http_err.reason)
- if ignoreError:
- return ''
- else:
- print 'Aborting.'
- sys.exit(1)
- except urllib2.URLError, url_err:
- print 'Error contacting %s: %s' % (resource, url_err.reason)
- if ignoreError:
- return ''
- else:
- raise url_err
- except httplib.BadStatusLine, err:
- if ignoreError:
- return ''
- else:
- raise err
- return response.read()
-
-
-# returns a map of (project, issue) => attachment id
-
-def parseJiraData(fileName):
- tree = ElementTree.parse(fileName)
- root = tree.getroot()
- jiraPattern = re.compile('([A-Z]+)\-([0-9]+)')
- result = {}
- for item in root.findall('./channel/item'):
- jirakey = item.find('key')
- if jirakey is None:
- continue
- jiraissue = jirakey.text
- matcher = jiraPattern.match(jiraissue)
- if not matcher:
- continue
- jiraissue = (matcher.group(1), matcher.group(2))
- attachmentIds = []
- for jiraattachment in item.findall('./attachments/attachment'):
- attachmentId = jiraattachment.get('id')
- try:
- attachmentIds.append(int(attachmentId))
- except ValueError:
- pass
- if attachmentIds:
- attachmentIds.sort()
- result[jiraissue] = attachmentIds[-1]
- return result
-
-
-if __name__ == '__main__':
- parser = OptionParser(prog = 'jenkins-admin')
- if os.getenv('JENKINS_URL'):
- parser.set_defaults(jenkinsUrl=os.getenv('JENKINS_URL'))
- if os.getenv('JOB_NAME'):
- parser.set_defaults(jenkinsJobName=os.getenv('JOB_NAME'))
- else:
- parser.set_defaults(jenkinsJobName='PreCommit-Admin')
-
- parser.set_defaults(jenkinsJobTemplate='PreCommit-{project}')
- parser.add_option('--initialize', action='store_true',
- dest='jenkinsInit',
- help='Start a new patch_tested.txt file')
- parser.add_option('--jenkins-jobname', type='string',
- dest='jenkinsJobName',
- help='PreCommit-Admin JobName', metavar='JOB_NAME')
- parser.add_option('--jenkins-project-template', type='string',
- dest='jenkinsJobTemplate',
- help='Template for project jobs',
- metavar='TEMPLATE')
- parser.add_option('--jenkins-token', type='string',
- dest='jenkinsToken', help='Jenkins Token',
- metavar='TOKEN')
- parser.add_option('--jenkins-url', type='string', dest='jenkinsUrl'
- , help='Jenkins base URL', metavar='URL')
- parser.add_option(
- '--jenkins-url-override',
- type='string',
- dest='jenkinsUrlOverrides',
- action='append',
- help='Project specific Jenkins base URL',
- metavar='PROJECT=URL',
- )
- parser.add_option('--jira-filter', type='string', dest='jiraFilter',
- help='JIRA filter URL', metavar='URL')
- parser.add_option('--jira-user', type='string', dest='jiraUser',
- help='JIRA username')
- parser.add_option('--jira-password', type='string', dest='jiraPassword',
- help='JIRA password')
- parser.add_option('--live', dest='live', action='store_true',
- help='Submit Job to jenkins')
- parser.add_option('--max-history', dest='history', type='int',
- help='Maximum history to store', default=5000)
- parser.add_option(
- '-V',
- '--version',
- dest='release_version',
- action='store_true',
- default=False,
- help="display version information for jenkins-admin and exit.")
-
- (options, args) = parser.parse_args()
-
- # Handle the version string right away and exit
- if options.release_version:
- with open(
- os.path.join(
- os.path.dirname(__file__), "../../VERSION"), 'r') as ver_file:
- print ver_file.read()
- sys.exit(0)
-
- tokenFrag = ''
- if options.jenkinsToken:
- tokenFrag = 'token=%s' % options.jenkinsToken
- else:
- tokenFrag = 'token={project}-token'
- if not options.jiraFilter:
- parser.error('ERROR: --jira-filter is a required argument.')
- if not options.jenkinsUrl:
- parser.error('ERROR: --jenkins-url or the JENKINS_URL environment variable is required.'
- )
- if options.history < 0:
- parser.error('ERROR: --max-history must be 0 or a positive integer.'
- )
- jenkinsUrlOverrides = {}
- if options.jenkinsUrlOverrides:
- for override in options.jenkinsUrlOverrides:
- if '=' not in override:
- parser.error('Invalid Jenkins Url Override: '
- + override)
- (project, url) = override.split('=', 1)
- jenkinsUrlOverrides[project.upper()] = url
- tempFile = NamedTemporaryFile(delete=False)
- try:
- jobLogHistory = None
- if not options.jenkinsInit:
- jobLogHistory = httpGet(options.jenkinsUrl
- + '/job/%s/lastSuccessfulBuild/artifact/patch_tested.txt'
- % options.jenkinsJobName, True)
-
- # if we don't have a successful build available try the last build
-
- if not jobLogHistory:
- jobLogHistory = httpGet(options.jenkinsUrl
- + '/job/%s/lastCompletedBuild/artifact/patch_tested.txt'
- % options.jenkinsJobName)
- jobLogHistory = jobLogHistory.strip().split('\n')
- if 'TESTED ISSUES' not in jobLogHistory[0]:
- print 'Downloaded patch_tested.txt control file may be corrupted. Failing.'
- sys.exit(1)
-
- # we are either going to write a new one or rewrite the old one
-
- jobLog = open('patch_tested.txt', 'w+')
-
- if jobLogHistory:
- if len(jobLogHistory) > options.history:
- jobLogHistory = [jobLogHistory[0]] \
- + jobLogHistory[len(jobLogHistory)
- - options.history:]
- for jobHistoryRecord in jobLogHistory:
- jobLog.write(jobHistoryRecord + '\n')
- else:
- jobLog.write('TESTED ISSUES\n')
- jobLog.flush()
- rssData = httpGet(options.jiraFilter,False,options.jiraUser,options.jiraPassword)
- tempFile.write(rssData)
- tempFile.flush()
- for (key, attachment) in parseJiraData(tempFile.name).items():
- (project, issue) = key
- if jenkinsUrlOverrides.has_key(project):
- url = jenkinsUrlOverrides[project]
- else:
- url = options.jenkinsUrl
-
- jenkinsUrlTemplate = url + '/job/' \
- + options.jenkinsJobTemplate \
- + '/buildWithParameters?' + tokenFrag \
- + '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}'
-
- urlArgs = {
- 'project': project,
- 'issue': issue,
- 'attachment': attachment,
- }
- jenkinsUrl = jenkinsUrlTemplate.format(**urlArgs)
-
- # submit job
-
- jobName = '%s-%s,%s' % (project, issue, attachment)
- if not jobLogHistory or jobName not in jobLogHistory:
- print jobName + ' has not been processed, submitting'
- jobLog.write(jobName + '\n')
- jobLog.flush()
- if options.live:
- httpGet(jenkinsUrl, True)
- else:
- print 'GET ' + jenkinsUrl
- else:
- print jobName + ' has been processed, ignoring'
- jobLog.close()
- finally:
- if options.live:
- os.remove(tempFile.name)
- else:
- print 'JIRA Data is located: ' + tempFile.name
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/jenkins/unit-test-filter-file.example
----------------------------------------------------------------------
diff --git a/precommit/jenkins/unit-test-filter-file.example b/precommit/jenkins/unit-test-filter-file.example
deleted file mode 100644
index dc38859..0000000
--- a/precommit/jenkins/unit-test-filter-file.example
+++ /dev/null
@@ -1,2 +0,0 @@
-[package].[class].[method]
-![package].[class].[method]
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/accumulo.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/accumulo.sh b/precommit/personality/accumulo.sh
deleted file mode 100755
index f2d3e7c..0000000
--- a/precommit/personality/accumulo.sh
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Run all plugins
-personality_plugins "all"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- # shellcheck disable=SC2034
- BUILDTOOL=maven
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=master
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^ACCUMULO-[0-9]+$'
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/accumulo"
- #shellcheck disable=SC2034
- PATCH_NAMING_RULE="http://accumulo.apache.org/git.html#contributors"
- # We want to invoke the 'check' goal, not the default 'checkstyle'. We define
- # our own checkstyle rules which isn't possible via 'checkstyle' configuration
- #shellcheck disable=SC2034
- CHECKSTYLE_GOAL="check"
-}
-
-## @description Queue up modules for this personality
-## @audience private
-## @stability evolving
-## @param repostatus
-## @param testtype
-function personality_modules
-{
- local repostatus=$1
- local testtype=$2
-
- yetus_debug "Personality: ${repostatus} ${testtype}"
- clear_personality_queue
-
- if [[ ${testtype} == 'unit' ]]; then
- # Run all tests, not just the tests in the modules affected
- yetus_debug "Overriding to run all unit tests"
-
- personality_enqueue_module .
- return
- fi
-
- # Make sure we re-add the changed modules if we didn't short-circuit out
- for module in "${CHANGED_MODULES[@]}"; do
- personality_enqueue_module "${module}"
- done
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/bigtop.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/bigtop.sh b/precommit/personality/bigtop.sh
deleted file mode 100755
index c374289..0000000
--- a/precommit/personality/bigtop.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all,-checkstyle,-findbugs"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- # shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=master
- # shellcheck disable=SC2034
- JIRA_ISSUE_RE='^BIGTOP-[0-9]+$'
- # shellcheck disable=SC2034
- BUILDTOOL=gradle
- # shellcheck disable=SC2034
- GITHUB_REPO="apache/bigtop"
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/drill.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/drill.sh b/precommit/personality/drill.sh
deleted file mode 100755
index cc35d45..0000000
--- a/precommit/personality/drill.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- # shellcheck disable=SC2034
- BUILDTOOL=maven
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/drill"
- #shellcheck disable=SC2034
- PATCH_NAMING_RULE="https://drill.apache.org/docs/apache-drill-contribution-guidelines/"
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^DRILL-[0-9]+$'
- #shellcheck disable=SC2034
- JIRA_STATUS_RE='Reviewable'
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/flink.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/flink.sh b/precommit/personality/flink.sh
deleted file mode 100755
index a9c0fdf..0000000
--- a/precommit/personality/flink.sh
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- # shellcheck disable=SC2034
- BUILDTOOL=maven
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=master
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^FLINK-[0-9]+$'
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/flink"
-}
-
-add_test_type flinklib
-
-## @description flinklib file filter
-## @audience private
-## @stability evolving
-## @param filename
-function flinklib_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.java$
- || ${filename} =~ \.scala$
- || ${filename} =~ pom.xml$ ]]; then
- add_test flinklib
- fi
-}
-
-## @description count files in the lib dir
-## @audience private
-## @stability evolving
-function flinklib_count
-{
- find "${BASEDIR}" \
- | ${GREP} "/lib/" \
- | ${GREP} -v "_qa_workdir" \
- | wc -l
-}
-
-## @description check fliblib
-## @audience private
-## @stability evolving
-## @param repostatus
-function flinklib_rebuild
-{
- declare repostatus=$1
-
- start_clock
- big_console_header "${PATCH_BRANCH} flink library dependencies"
-
- if ! verify_needed_test flinklib; then
- echo "${BUILDMODEMSG} does not need flinklib testing."
- return 0
- fi
-
- pushd "${BASEDIR}" >/dev/null
- echo_and_redirect "${PATCH_DIR}/${repostatus}-flinklib-root.txt" \
- "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
- if [[ $? != 0 ]]; then
- add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}."
- fi
-
- if [[ ${repostatus} = branch ]]; then
- FLINK_PRE_LIB_FILES=$(flinklib_count)
- return 0
- else
- FLINK_POST_LIB_FILES=$(flinklib_count)
- fi
- popd >/dev/null
-
- if [[ "${BUILDMODE}" = full ]]; then
- if [[ ${FLINK_POST_LIB_FILES} -gt 0 ]]; then
- add_vote_table -1 flinklib "Lib folder dependencies are currently ${FLINK_POST_LIB_FILES}"
- return 1
- else
- add_vote_table +1 flinklib "No lib folder dependencies!"
- return 0
- fi
- fi
-
- if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then
- add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \
- "${FLINK_PRE_LIB_FILES} to ${FLINK_POST_LIB_FILES}"
- return 1
- elif [[ "${FLINK_POST_LIB_FILES}" -eq "${FLINK_PRE_LIB_FILES}" ]]; then
- add_vote_table 0 flinklib "Patch did not change lib dependencies" \
- " (still ${FLINK_PRE_LIB_FILES})"
- else
- add_vote_table +1 flinklib "Patch decreases lib folder dependencies by " \
- "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))."
- fi
- return 0
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/geode.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/geode.sh b/precommit/personality/geode.sh
deleted file mode 100755
index 9f5c0ec..0000000
--- a/precommit/personality/geode.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all,-ant,-javac,-scalac,-scaladoc"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=develop
- #shellcheck disable=SC2034
- PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/GEODE/How+to+Contribute"
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^(GEODE)-[0-9]+$'
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/incubator-geode"
- #shellcheck disable=SC2034
- BUILDTOOL=gradle
-# PYLINT_OPTIONS="--indent-string=' '"
-
-# HADOOP_MODULES=""
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh
deleted file mode 100755
index 2ad4459..0000000
--- a/precommit/personality/hadoop.sh
+++ /dev/null
@@ -1,563 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# SHELLDOC-IGNORE
-#
-# Override these to match Apache Hadoop's requirements
-
-personality_plugins "all,-ant,-gradle,-scalac,-scaladoc"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- # shellcheck disable=SC2034
- BUILDTOOL=maven
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=trunk
- #shellcheck disable=SC2034
- PATCH_NAMING_RULE="https://wiki.apache.org/hadoop/HowToContribute"
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS|HDDS)-[0-9]+$'
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/hadoop"
- #shellcheck disable=SC2034
- PYLINT_OPTIONS="--indent-string=' '"
-
- HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)}
- if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then
- HADOOP_HOMEBREW_DIR=/usr/local
- fi
-}
-
-function personality_parse_args
-{
- declare i
-
- for i in "$@"; do
- case ${i} in
- --hadoop-isal-prefix=*)
- ISAL_HOME=${i#*=}
- ;;
- --hadoop-openssl-prefix=*)
- OPENSSL_HOME=${i#*=}
- ;;
- --hadoop-snappy-prefix=*)
- SNAPPY_HOME=${i#*=}
- ;;
- esac
- done
-}
-
-## @description Calculate the actual module ordering
-## @audience private
-## @stability evolving
-## @param ordering
-function hadoop_order
-{
- declare ordering=$1
- declare hadoopm
-
- if [[ ${ordering} = normal ]]; then
- hadoopm="${CHANGED_MODULES[*]}"
- elif [[ ${ordering} = union ]]; then
- hadoopm="${CHANGED_UNION_MODULES}"
- elif [[ ${ordering} = mvnsrc ]]; then
- hadoopm="${MAVEN_SRC_MODULES[*]}"
- elif [[ ${ordering} = mvnsrctest ]]; then
- hadoopm="${MAVEN_SRCTEST_MODULES[*]}"
- else
- hadoopm="${ordering}"
- fi
- echo "${hadoopm}"
-}
-
-## @description Determine if it is safe to run parallel tests
-## @audience private
-## @stability evolving
-## @param ordering
-function hadoop_test_parallel
-{
- if [[ -f "${BASEDIR}/pom.xml" ]]; then
- HADOOP_VERSION=$(grep '<version>' "${BASEDIR}/pom.xml" \
- | head -1 \
- | "${SED}" -e 's|^ *<version>||' -e 's|</version>.*$||' \
- | cut -f1 -d- )
- export HADOOP_VERSION
- else
- return 1
- fi
-
- hmajor=${HADOOP_VERSION%%\.*}
- hmajorminor=${HADOOP_VERSION%\.*}
- hminor=${hmajorminor##*\.}
- # ... and just for reference
- #hmicro=${HADOOP_VERSION##*\.}
-
- # Apache Hadoop v2.8.0 was the first one to really
- # get working parallel unit tests
- if [[ ${hmajor} -lt 3 && ${hminor} -lt 8 ]]; then
- return 1
- fi
-
- return 0
-}
-
-## @description Install extra modules for unit tests
-## @audience private
-## @stability evolving
-## @param ordering
-function hadoop_unittest_prereqs
-{
- declare input=$1
- declare mods
- declare need_common=0
- declare building_common=0
- declare module
- declare flags
- declare fn
-
- # prior to running unit tests, hdfs needs libhadoop.so built
- # if we're building root, then this extra work is moot
-
- #shellcheck disable=SC2086
- mods=$(hadoop_order ${input})
-
- for module in ${mods}; do
- if [[ ${module} = hadoop-hdfs-project* ]]; then
- need_common=1
- elif [[ ${module} = hadoop-common-project/hadoop-common
- || ${module} = hadoop-common-project ]]; then
- building_common=1
- elif [[ ${module} = . ]]; then
- return
- fi
- done
-
- # Windows builds *ALWAYS* need hadoop-common compiled
- case ${OSTYPE} in
- Windows_NT|CYGWIN*|MINGW*|MSYS*)
- need_common=1
- ;;
- esac
-
- if [[ ${need_common} -eq 1
- && ${building_common} -eq 0 ]]; then
- echo "unit test pre-reqs:"
- module="hadoop-common-project/hadoop-common"
- fn=$(module_file_fragment "${module}")
- flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
- pushd "${BASEDIR}/${module}" >/dev/null
- # shellcheck disable=SC2086
- echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \
- "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
- popd >/dev/null
- fi
-}
-
-## @description Calculate the flags/settings for yarn-ui v2 build
-## @description based upon the OS
-## @audience private
-## @stability evolving
-function yarn_ui2_flag
-{
-
- if [[ ${BUILD_NATIVE} != true ]]; then
- return
- fi
-
- # Now it only tested on Linux/OSX, don't enable the profile on
- # windows until it get verified
- case ${OSTYPE} in
- Linux)
- # shellcheck disable=SC2086
- echo -Pyarn-ui
- ;;
- Darwin)
- echo -Pyarn-ui
- ;;
- *)
- # Do nothing
- ;;
- esac
-}
-
-## @description Calculate the flags/settings for native code
-## @description based upon the OS
-## @audience private
-## @stability evolving
-function hadoop_native_flags
-{
- if [[ ${BUILD_NATIVE} != true ]]; then
- return
- fi
-
- declare -a args
-
- # Based upon HADOOP-11937
- #
- # Some notes:
- #
- # - getting fuse to compile on anything but Linux
- # is always tricky.
- # - Darwin assumes homebrew is in use.
- # - HADOOP-12027 required for bzip2 on OS X.
- # - bzip2 is broken in lots of places
- # (the shared library is considered experimental)
- # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2
- #
-
- args=("-Drequire.test.libhadoop")
-
- if [[ -d "${ISAL_HOME}/include" ]]; then
- args=("${args[@]}" "-Disal.prefix=${ISAL_HOME}")
- fi
-
- if [[ -d "${OPENSSL_HOME}/include" ]]; then
- args=("${args[@]}" "-Dopenssl.prefix=${OPENSSL_HOME}")
- elif [[ -d "${HADOOP_HOMEBREW_DIR}/opt/openssl/" ]]; then
- args=("${args[@]}" "-Dopenssl.prefix=${HADOOP_HOMEBREW_DIR}/opt/openssl/")
- fi
-
- if [[ -d "${SNAPPY_HOME}/include" ]]; then
- args=("${args[@]}" "-Dsnappy.prefix=${SNAPPY_HOME}")
- elif [[ -d "${HADOOP_HOMEBREW_DIR}/include/snappy.h" ]]; then
- args=("${args[@]}" "-Dsnappy.prefix=${HADOOP_HOMEBREW_DIR}/opt/snappy")
- fi
-
- case ${OSTYPE} in
- Linux)
- # shellcheck disable=SC2086
- echo \
- -Pnative \
- -Drequire.fuse \
- -Drequire.openssl \
- -Drequire.snappy \
- -Drequire.valgrind \
- "${args[@]}"
- ;;
- Darwin)
- echo \
- "${args[@]}" \
- -Pnative \
- -Drequire.snappy \
- -Drequire.openssl
- ;;
- Windows_NT|CYGWIN*|MINGW*|MSYS*)
- echo \
- "${args[@]}" \
- -Drequire.snappy -Drequire.openssl -Pnative-win
- ;;
- *)
- echo \
- "${args[@]}"
- ;;
- esac
-}
-
-## @description Queue up modules for this personality
-## @audience private
-## @stability evolving
-## @param repostatus
-## @param testtype
-function personality_modules
-{
- declare repostatus=$1
- declare testtype=$2
- declare extra=""
- declare ordering="normal"
- declare needflags=false
- declare foundbats=false
- declare flags
- declare fn
- declare i
- declare hadoopm
-
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- case ${testtype} in
- asflicense)
- # this is very fast and provides the full path if we do it from
- # the root of the source
- personality_enqueue_module .
- return
- ;;
- checkstyle)
- ordering="union"
- extra="-DskipTests"
- ;;
- compile)
- ordering="union"
- extra="-DskipTests"
- needflags=true
-
- # if something in common changed, we build the whole world
- if [[ "${CHANGED_MODULES[*]}" =~ hadoop-common ]]; then
- yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . "
- ordering="."
- fi
- ;;
- distclean)
- ordering="."
- extra="-DskipTests"
- ;;
- javadoc)
- if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
- ordering=.
- fi
-
- if [[ "${repostatus}" = patch && "${BUILDMODE}" = patch ]]; then
- echo "javadoc pre-reqs:"
- for i in hadoop-project \
- hadoop-common-project/hadoop-annotations; do
- fn=$(module_file_fragment "${i}")
- pushd "${BASEDIR}/${i}" >/dev/null
- echo "cd ${i}"
- echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \
- "${MAVEN}" "${MAVEN_ARGS[@]}" install
- popd >/dev/null
- done
- fi
- extra="-Pdocs -DskipTests"
- ;;
- mvneclipse)
- if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
- ordering=.
- fi
- ;;
- mvninstall)
- extra="-DskipTests"
- if [[ "${repostatus}" = branch || "${BUILDMODE}" = full ]]; then
- ordering=.
- fi
- ;;
- mvnsite)
- if [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
- ordering=.
- fi
- ;;
- unit)
- if [[ "${BUILDMODE}" = full ]]; then
- ordering=mvnsrc
- elif [[ "${CHANGED_MODULES[*]}" =~ \. ]]; then
- ordering=.
- fi
-
- if [[ ${TEST_PARALLEL} = "true" ]] ; then
- if hadoop_test_parallel; then
- extra="-Pparallel-tests"
- if [[ -n ${TEST_THREADS:-} ]]; then
- extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
- fi
- fi
- fi
- needflags=true
- hadoop_unittest_prereqs "${ordering}"
-
- if ! verify_needed_test javac; then
- yetus_debug "hadoop: javac not requested"
- if ! verify_needed_test native; then
- yetus_debug "hadoop: native not requested"
- yetus_debug "hadoop: adding -DskipTests to unit test"
- extra="-DskipTests"
- fi
- fi
-
- for i in "${CHANGED_FILES[@]}"; do
- if [[ "${i}" =~ \.bats ]]; then
- foundbats=true
- fi
- done
-
- if ! verify_needed_test shellcheck && [[ ${foundbats} = false ]]; then
- yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile"
- extra="${extra} -P!shelltest"
- else
- extra="${extra} -Pshelltest"
- fi
- ;;
- *)
- extra="-DskipTests"
- ;;
- esac
-
- if [[ ${needflags} = true ]]; then
- flags="$(hadoop_native_flags) $(yarn_ui2_flag)"
- extra="${extra} ${flags}"
- fi
-
- extra="-Ptest-patch ${extra}"
-
- for module in $(hadoop_order ${ordering}); do
- # shellcheck disable=SC2086
- personality_enqueue_module ${module} ${extra}
- done
-}
-
-## @description Add tests based upon personality needs
-## @audience private
-## @stability evolving
-## @param filename
-function personality_file_tests
-{
- declare filename=$1
-
- yetus_debug "Using Hadoop-specific personality_file_tests"
-
- if [[ ${filename} =~ src/main/webapp ]]; then
- yetus_debug "tests/webapp: ${filename}"
- add_test shadedclient
- elif [[ ${filename} =~ \.sh
- || ${filename} =~ \.cmd
- || ${filename} =~ src/scripts
- || ${filename} =~ src/test/scripts
- || ${filename} =~ src/main/bin
- || ${filename} =~ shellprofile\.d
- || ${filename} =~ src/main/conf
- ]]; then
- yetus_debug "tests/shell: ${filename}"
- add_test mvnsite
- add_test unit
- elif [[ ${filename} =~ \.md$
- || ${filename} =~ \.md\.vm$
- || ${filename} =~ src/site
- ]]; then
- yetus_debug "tests/site: ${filename}"
- add_test mvnsite
- elif [[ ${filename} =~ \.c$
- || ${filename} =~ \.cc$
- || ${filename} =~ \.h$
- || ${filename} =~ \.hh$
- || ${filename} =~ \.proto$
- || ${filename} =~ \.cmake$
- || ${filename} =~ CMakeLists.txt
- ]]; then
- yetus_debug "tests/units: ${filename}"
- add_test compile
- add_test cc
- add_test mvnsite
- add_test javac
- add_test unit
- elif [[ ${filename} =~ build.xml$
- || ${filename} =~ pom.xml$
- || ${filename} =~ \.java$
- || ${filename} =~ src/main
- ]]; then
- yetus_debug "tests/javadoc+units: ${filename}"
- add_test compile
- add_test javac
- add_test javadoc
- add_test mvninstall
- add_test mvnsite
- add_test unit
- add_test shadedclient
- fi
-
- # if we change anything in here, e.g. the test scripts
- # then run the client artifact tests
- if [[ ${filename} =~ hadoop-client-modules ]]; then
- add_test shadedclient
- fi
-
- if [[ ${filename} =~ src/test ]]; then
- yetus_debug "tests: src/test"
- add_test unit
- fi
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test findbugs
- fi
-}
-
-## @description Image to print on success
-## @audience private
-## @stability evolving
-function hadoop_console_success
-{
- printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
- printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
- printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
- printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
- printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
- printf "ICB8bXwgICB8bXwgIAo"
-}
-
-###################################################
-# Hadoop project specific check of IT for shaded artifacts
-
-add_test_type shadedclient
-
-## @description check for test modules and add test/plugins as needed
-## @audience private
-## @stability evolving
-function shadedclient_initialize
-{
- maven_add_install shadedclient
-}
-
-## @description build client facing shaded artifacts and test them
-## @audience private
-## @stability evolving
-## @param repostatus
-function shadedclient_rebuild
-{
- declare repostatus=$1
- declare logfile="${PATCH_DIR}/${repostatus}-shadedclient.txt"
- declare module
- declare -a modules=()
-
- if [[ ${OSTYPE} = Windows_NT ||
- ${OSTYPE} =~ ^CYGWIN.* ||
- ${OSTYPE} =~ ^MINGW32.* ||
- ${OSTYPE} =~ ^MSYS.* ]]; then
- echo "hadoop personality: building on windows, skipping check of client artifacts."
- return 0
- fi
-
- yetus_debug "hadoop personality: seeing if we need the test of client artifacts."
- for module in hadoop-client-modules/hadoop-client-check-invariants \
- hadoop-client-modules/hadoop-client-check-test-invariants \
- hadoop-client-modules/hadoop-client-integration-tests; do
- if [ -d "${module}" ]; then
- yetus_debug "hadoop personality: test module '${module}' is present."
- modules=( ${modules[@]} -pl ${module} )
- fi
- done
- if [ ${#modules[@]} -eq 0 ]; then
- echo "hadoop personality: no test modules present, skipping check of client artifacts."
- return 0
- fi
-
- big_console_header "Checking client artifacts on ${repostatus}"
-
- echo_and_redirect "${logfile}" \
- "${MAVEN}" "${MAVEN_ARGS[@]}" verify -fae --batch-mode -am \
- "${modules[@]}" \
- -Dtest=NoUnitTests -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true
-
- count=$(${GREP} -c '\[ERROR\]' "${logfile}")
- if [[ ${count} -gt 0 ]]; then
- add_vote_table -1 shadedclient "${repostatus} has errors when building and testing our client artifacts."
- return 1
- fi
-
- add_vote_table +1 shadedclient "${repostatus} has no errors when building and testing our client artifacts."
- return 0
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/hbase.sh b/precommit/personality/hbase.sh
deleted file mode 100755
index b48691e..0000000
--- a/precommit/personality/hbase.sh
+++ /dev/null
@@ -1,368 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- BUILDTOOL=maven
- #shellcheck disable=SC2034
- PROJECT_NAME=hbase
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=master
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^HBASE-[0-9]+$'
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/hbase"
-
- # TODO use PATCH_BRANCH to select hadoop versions to use.
- # All supported Hadoop versions that we want to test the compilation with
- HBASE_HADOOP_VERSIONS="2.4.0 2.4.1 2.5.0 2.5.1 2.5.2 2.6.1 2.6.2 2.6.3 2.7.1"
-
- # TODO use PATCH_BRANCH to select jdk versions to use.
-
- # Override the maven options
- MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
-
-}
-
-## @description Queue up modules for this personality
-## @audience private
-## @stability evolving
-## @param repostatus
-## @param testtype
-function personality_modules
-{
- local repostatus=$1
- local testtype=$2
- local extra=""
-
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- extra="-DHBasePatchProcess"
-
- if [[ ${repostatus} == branch
- && ${testtype} == mvninstall ]] ||
- [[ "${BUILDMODE}" == full ]];then
- personality_enqueue_module . ${extra}
- return
- fi
-
- if [[ ${testtype} = findbugs ]]; then
- for module in "${CHANGED_MODULES[@]}"; do
- # skip findbugs on hbase-shell and hbase-it. hbase-it has nothing
- # in src/main/java where findbugs goes to look
- if [[ ${module} == hbase-shell ]]; then
- continue
- elif [[ ${module} == hbase-it ]]; then
- continue
- else
- # shellcheck disable=SC2086
- personality_enqueue_module ${module} ${extra}
- fi
- done
- return
- fi
-
- if [[ ${testtype} = unit ]]; then
- extra="${extra} -PrunAllTests"
-
- # Inject the jenkins build-id for our surefire invocations
- # Used by zombie detection stuff, even though we're not including that yet.
- if [ -n "${BUILD_ID}" ]; then
- extra="${extra} -Dbuild.id=${BUILD_ID}"
- fi
- fi
-
- for module in "${CHANGED_MODULES[@]}"; do
- # shellcheck disable=SC2086
- personality_enqueue_module ${module} ${extra}
- done
-}
-
-###################################################
-# Below here are our one-off tests specific to hbase.
-# TODO break them into individual files so it's easier to maintain them?
-
-# TODO line length check? could ignore all java files since checkstyle gets them.
-
-###################################################
-
-add_test_type hadoopcheck
-
-## @description hadoopcheck file filter
-## @audience private
-## @stability evolving
-## @param filename
-function hadoopcheck_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test hadoopcheck
- fi
-}
-
-## @description hadoopcheck test
-## @audience private
-## @stability evolving
-## @param repostatus
-function hadoopcheck_rebuild
-{
- local repostatus=$1
- local hadoopver
- local logfile
- local count
- local result=0
-
- if [[ "${repostatus}" = branch ]]; then
- return 0
- fi
-
- big_console_header "Compiling against various Hadoop versions"
-
- export MAVEN_OPTS="${MAVEN_OPTS}"
- for hadoopver in ${HBASE_HADOOP_VERSIONS}; do
- logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
- echo_and_redirect "${logfile}" \
- "${MAVEN}" clean install \
- -DskipTests -DHBasePatchProcess \
- -Dhadoop-two.version="${hadoopver}"
- count=$(${GREP} -c ERROR "${logfile}")
- if [[ ${count} -gt 0 ]]; then
- add_vote_table -1 hadoopcheck "${BUILDMODEMSG} causes ${count} errors with Hadoop v${hadoopver}."
- ((result=result+1))
- fi
- done
-
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
-
- add_vote_table +1 hadoopcheck "${BUILDMODEMSG} does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
- return 0
-}
-
-######################################
-
-# TODO if we need the protoc check, we probably need to check building all the modules that rely on hbase-protocol
-add_test_type hbaseprotoc
-
-## @description hbaseprotoc file filter
-## @audience private
-## @stability evolving
-## @param filename
-function hbaseprotoc_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.proto$ ]]; then
- add_test hbaseprotoc
- fi
-}
-
-## @description hadoopcheck test
-## @audience private
-## @stability evolving
-## @param repostatus
-function hbaseprotoc_rebuild
-{
- declare repostatus=$1
- declare i=0
- declare fn
- declare module
- declare logfile
- declare count
- declare result
-
- if [[ "${repostatus}" = branch ]]; then
- return 0
- fi
-
- if ! verify_needed_test hbaseprotoc; then
- return 0
- fi
-
- big_console_header "HBase protoc plugin: ${BUILDMODE}"
-
- start_clock
-
- personality_modules patch hbaseprotoc
- modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
-
- # shellcheck disable=SC2153
- until [[ $i -eq "${#MODULE[@]}" ]]; do
- if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
- ((result=result+1))
- ((i=i+1))
- continue
- fi
- module=${MODULE[$i]}
- fn=$(module_file_fragment "${module}")
- logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
-
- count=$(${GREP} -c ERROR "${logfile}")
-
- if [[ ${count} -gt 0 ]]; then
- module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
- "${count} new protoc errors in ${module}."
- ((result=result+1))
- fi
- ((i=i+1))
- done
-
- modules_messages patch hbaseprotoc true
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
- return 0
-}
-
-######################################
-
-add_test_type hbaseanti
-
-## @description hbaseanti file filter
-## @audience private
-## @stability evolving
-## @param filename
-function hbaseanti_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test hbaseanti
- fi
-}
-
-## @description hbaseanti patch file check
-## @audience private
-## @stability evolving
-## @param filename
-function hbaseanti_patchfile
-{
- local patchfile=$1
- local warnings
- local result
-
- if [[ "${BUILDMODE}" = full ]]; then
- return 0
- fi
-
- if ! verify_needed_test hbaseanti; then
- return 0
- fi
-
- big_console_header "Checking for known anti-patterns"
-
- start_clock
-
- warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}")
- if [[ ${warnings} -gt 0 ]]; then
- add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}."
- ((result=result+1))
- fi
-
- warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}")
- if [[ ${warnings} -gt 0 ]]; then
- add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}."
- ((result=result+1))
- fi
-
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
-
- add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
- return 0
-}
-
-
-## @description hbase custom mvnsite file filter. See HBASE-15042
-## @audience private
-## @stability evolving
-## @param filename
-function mvnsite_filefilter
-{
- local filename=$1
-
- if [[ ${BUILDTOOL} = maven ]]; then
- if [[ ${filename} =~ src/main/site || ${filename} =~ src/main/asciidoc ]]; then
- yetus_debug "tests/mvnsite: ${filename}"
- add_test mvnsite
- fi
- fi
-}
-
-## This is named so that yetus will check us right after running tests.
-## Essentially, we check for normal failures and then we look for zombies.
-#function hbase_unit_logfilter
-#{
-# declare testtype="unit"
-# declare input=$1
-# declare output=$2
-# declare processes
-# declare process_output
-# declare zombies
-# declare zombie_count=0
-# declare zombie_process
-#
-# yetus_debug "in hbase-specific unit logfilter."
-#
-# # pass-through to whatever is counting actual failures
-# if declare -f ${BUILDTOOL}_${testtype}_logfilter >/dev/null; then
-# "${BUILDTOOL}_${testtype}_logfilter" "${input}" "${output}"
-# elif declare -f ${testtype}_logfilter >/dev/null; then
-# "${testtype}_logfilter" "${input}" "${output}"
-# fi
-#
-# start_clock
-# if [ -n "${BUILD_ID}" ]; then
-# yetus_debug "Checking for zombie test processes."
-# processes=$(jps -v | "${GREP}" surefirebooter | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
-# if [ -n "${processes}" ] && [ "$(echo "${processes}" | wc -l)" -gt 0 ]; then
-# yetus_warn "Found some suspicious process(es). Waiting a bit to see if they're just slow to stop."
-# yetus_debug "${processes}"
-# sleep 30
-# #shellcheck disable=SC2016
-# for pid in $(echo "${processes}"| ${AWK} '{print $1}'); do
-# # Test our zombie still running (and that it still an hbase build item)
-# process_output=$(ps -p "${pid}" | tail +2 | "${GREP}" -e "hbase.build.id=${BUILD_ID}")
-# if [[ -n "${process_output}" ]]; then
-# yetus_error "Zombie: ${process_output}"
-# ((zombie_count = zombie_count + 1))
-# zombie_process=$(jstack "${pid}" | "${GREP}" -e "\.Test" | "${GREP}" -e "\.java"| head -3)
-# zombies="${zombies} ${zombie_process}"
-# fi
-# done
-# fi
-# if [ "${zombie_count}" -ne 0 ]; then
-# add_vote_table -1 zombies "There are ${zombie_count} zombie test(s)"
-# populate_test_table "zombie unit tests" "${zombies}"
-# else
-# yetus_info "Zombie check complete. All test runs exited normally."
-# stop_clock
-# fi
-# else
-# add_vote_table -0 zombies "There is no BUILD_ID env variable; can't check for zombies."
-# fi
-#
-#}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/jmeter.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/jmeter.sh b/precommit/personality/jmeter.sh
deleted file mode 100755
index b4e61b5..0000000
--- a/precommit/personality/jmeter.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all,-javadoc,-findbugs,-asflicense"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- # shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=trunk
- # shellcheck disable=SC2034
- BUILDTOOL=ant
- # shellcheck disable=SC2034
- GITHUB_REPO="apache/jmeter"
- # shellcheck disable=SC2034
- JMETER_DOWNLOAD_JARS=false
-}
-
-add_test_type jmeter
-
-## @description Personality usage options
-## @audience private
-## @stability evolving
-function jmeter_usage
-{
- yetus_add_option "--jmeter-download-jars=<bool>" "download third-party jars needed by ant build"
-}
-
-## @description Process personality options
-## @audience private
-## @stability evolving
-## @param arguments
-function jmeter_parse_args
-{
- declare i
-
- for i in "$@"; do
- case ${i} in
- --jmeter-download-jars=*)
- JMETER_DOWNLOAD_JARS=${i#*=}
- ;;
- esac
- done
-}
-
-## @description Download jmetere dependencies
-## @audience private
-## @stability evolving
-function jmeter_precheck
-{
- if [[ ${JMETER_DOWNLOAD_JARS} = true ]]; then
- pushd "${BASEDIR}" >/dev/null
- echo_and_redirect "${PATCH_DIR}/jmeter-branch-download-jars.txt" "${ANT}" download_jars
- popd >/dev/null
- fi
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/kafka.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/kafka.sh b/precommit/personality/kafka.sh
deleted file mode 100755
index 0c2f749..0000000
--- a/precommit/personality/kafka.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all,-checkstyle,-asflicense"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=trunk
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^KAFKA-[0-9]+$'
- #shellcheck disable=SC2034
- PATCH_NAMING_RULE="http://kafka.apache.org/contributing.html"
- # shellcheck disable=SC2034
- BUILDTOOL=gradle
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/kafka"
-}
-
-## @description Queue up modules for this personality
-## @audience private
-## @stability evolving
-## @param repostatus
-## @param testtype
-function personality_modules
-{
- declare repostatus=$1
- declare testtype=$2
- declare module
- declare extra=""
-
- yetus_debug "Using kafka personality_modules"
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- case ${testtype} in
- gradleboot)
- # kafka's bootstrap is broken
- if [[ ${testtype} == gradleboot ]]; then
- pushd "${BASEDIR}" >/dev/null
- echo_and_redirect "${PATCH_DIR}/kafka-configure-gradle.txt" gradle
- popd >/dev/null
- fi
- ;;
- compile)
- extra="clean jar"
- ;;
- esac
-
- for module in "${CHANGED_MODULES[@]}"; do
- # shellcheck disable=SC2086
- personality_enqueue_module "${module}" ${extra}
- done
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/orc.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/orc.sh b/precommit/personality/orc.sh
deleted file mode 100755
index 0582b17..0000000
--- a/precommit/personality/orc.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all,-checkstyle,-findbugs"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=master
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^ORC-[0-9]+$'
- # shellcheck disable=SC2034
- BUILDTOOL=cmake
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/orc"
-
- # protobuf fails
- #shellcheck disable=SC2034
- CMAKE_ROOT_BUILD=true
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/pig.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/pig.sh b/precommit/personality/pig.sh
deleted file mode 100755
index 2014fda..0000000
--- a/precommit/personality/pig.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=trunk
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^PIG-[0-9]+$'
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/pig"
- #shellcheck disable=SC2034
- BUILDTOOL=ant
-}
-
-## @description Queue up modules for this personality
-## @audience private
-## @stability evolving
-## @param repostatus
-## @param testtype
-function personality_modules
-{
- local repostatus=$1
- local testtype=$2
- local extra=""
-
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- extra="-DPigPatchProcess= "
-
- case ${testtype} in
- findbugs)
- # shellcheck disable=SC2034
- ANT_FINDBUGSXML="${BASEDIR}/build/test/findbugs/pig-findbugs-report.xml"
- extra="-Dfindbugs.home=${FINDBUGS_HOME}"
- ;;
- compile)
- extra="${extra} -Djavac.args=-Xlint -Dcompile.c++=yes clean piggybank"
- ;;
- javadoc)
- extra="${extra} -Dforrest.home=${FORREST_HOME}"
- ;;
- unit)
- extra="${extra} -Dtest.junit.output.format=xml -Dcompile.c++=yes -Dtest.output=yes test-core"
- ;;
- esac
-
- # shellcheck disable=SC2086
- personality_enqueue_module . ${extra}
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/ratis.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/ratis.sh b/precommit/personality/ratis.sh
deleted file mode 100755
index d700512..0000000
--- a/precommit/personality/ratis.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- # shellcheck disable=SC2034
- BUILDTOOL=maven
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=master
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^RATIS-[0-9]+$'
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/incubator-ratis"
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/personality/samza.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/samza.sh b/precommit/personality/samza.sh
deleted file mode 100755
index 430ffbb..0000000
--- a/precommit/personality/samza.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-personality_plugins "all,-checkstyle"
-
-## @description Globals specific to this personality
-## @audience private
-## @stability evolving
-function personality_globals
-{
- #shellcheck disable=SC2034
- PATCH_BRANCH_DEFAULT=master
- #shellcheck disable=SC2034
- JIRA_ISSUE_RE='^SAMZA-[0-9]+$'
- #shellcheck disable=SC2034
- PATCH_NAMING_RULE="https://cwiki.apache.org/confluence/display/SAMZA/Contributor's+Corner"
- # shellcheck disable=SC2034
- BUILDTOOL=gradle
- #shellcheck disable=SC2034
- GITHUB_REPO="apache/samza"
-}