You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/01/21 21:22:16 UTC
[1/2] hadoop git commit: HADOOP-12651. Replace dev-support with
wrappers to Yetus (aw)
Repository: hadoop
Updated Branches:
refs/heads/trunk c304890c8 -> 8cecad2d5
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
deleted file mode 100755
index 3c52912..0000000
--- a/dev-support/test-patch.sh
+++ /dev/null
@@ -1,2814 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-### BUILD_URL is set by Hudson if it is run by patch process
-
-this="${BASH_SOURCE-$0}"
-BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-CWD=$(pwd)
-USER_PARAMS=("$@")
-GLOBALTIMER=$(date +"%s")
-
-## @description Setup the default global variables
-## @audience public
-## @stability stable
-## @replaceable no
-function setup_defaults
-{
- if [[ -z "${MAVEN_HOME:-}" ]]; then
- MVN=mvn
- else
- MVN=${MAVEN_HOME}/bin/mvn
- fi
- # This parameter needs to be kept as an array
- MAVEN_ARGS=()
-
- PROJECT_NAME=hadoop
- HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
- JENKINS=false
- BASEDIR=$(pwd)
- RELOCATE_PATCH_DIR=false
-
- USER_PLUGIN_DIR=""
- LOAD_SYSTEM_PLUGINS=true
-
- FINDBUGS_HOME=${FINDBUGS_HOME:-}
- FINDBUGS_WARNINGS_FAIL_PRECHECK=false
- ECLIPSE_HOME=${ECLIPSE_HOME:-}
- BUILD_NATIVE=${BUILD_NATIVE:-true}
- PATCH_BRANCH=""
- PATCH_BRANCH_DEFAULT="trunk"
- CHANGED_MODULES=""
- USER_MODULE_LIST=""
- OFFLINE=false
- CHANGED_FILES=""
- REEXECED=false
- RESETREPO=false
- ISSUE=""
- ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
- TIMER=$(date +"%s")
- PATCHURL=""
-
- OSTYPE=$(uname -s)
-
- # Solaris needs POSIX, not SVID
- case ${OSTYPE} in
- SunOS)
- PS=${PS:-ps}
- AWK=${AWK:-/usr/xpg4/bin/awk}
- SED=${SED:-/usr/xpg4/bin/sed}
- WGET=${WGET:-wget}
- GIT=${GIT:-git}
- EGREP=${EGREP:-/usr/xpg4/bin/egrep}
- GREP=${GREP:-/usr/xpg4/bin/grep}
- PATCH=${PATCH:-patch}
- DIFF=${DIFF:-/usr/gnu/bin/diff}
- JIRACLI=${JIRA:-jira}
- FILE=${FILE:-file}
- ;;
- *)
- PS=${PS:-ps}
- AWK=${AWK:-awk}
- SED=${SED:-sed}
- WGET=${WGET:-wget}
- GIT=${GIT:-git}
- EGREP=${EGREP:-egrep}
- GREP=${GREP:-grep}
- PATCH=${PATCH:-patch}
- DIFF=${DIFF:-diff}
- JIRACLI=${JIRA:-jira}
- FILE=${FILE:-file}
- ;;
- esac
-
- declare -a JIRA_COMMENT_TABLE
- declare -a JIRA_FOOTER_TABLE
- declare -a JIRA_HEADER
- declare -a JIRA_TEST_TABLE
-
- JFC=0
- JTC=0
- JTT=0
- RESULT=0
-}
-
-## @description Print a message to stderr
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function hadoop_error
-{
- echo "$*" 1>&2
-}
-
-## @description Print a message to stderr if --debug is turned on
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function hadoop_debug
-{
- if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
- echo "[$(date) DEBUG]: $*" 1>&2
- fi
-}
-
-## @description Activate the local timer
-## @audience public
-## @stability stable
-## @replaceable no
-function start_clock
-{
- hadoop_debug "Start clock"
- TIMER=$(date +"%s")
-}
-
-## @description Print the elapsed time in seconds since the start of the local timer
-## @audience public
-## @stability stable
-## @replaceable no
-function stop_clock
-{
- local -r stoptime=$(date +"%s")
- local -r elapsed=$((stoptime-TIMER))
- hadoop_debug "Stop clock"
-
- echo ${elapsed}
-}
-
-## @description Print the elapsed time in seconds since the start of the global timer
-## @audience private
-## @stability stable
-## @replaceable no
-function stop_global_clock
-{
- local -r stoptime=$(date +"%s")
- local -r elapsed=$((stoptime-GLOBALTIMER))
- hadoop_debug "Stop global clock"
-
- echo ${elapsed}
-}
-
-## @description Add time to the local timer
-## @audience public
-## @stability stable
-## @replaceable no
-## @param seconds
-function offset_clock
-{
- ((TIMER=TIMER-$1))
-}
-
-## @description Add to the header of the display
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-function add_jira_header
-{
- JIRA_HEADER[${JHC}]="| $* |"
- JHC=$(( JHC+1 ))
-}
-
-## @description Add to the output table. If the first parameter is a number
-## @description that is the vote for that column and calculates the elapsed time
-## @description based upon the last start_clock(). If it the string null, then it is
-## @description a special entry that signifies extra
-## @description content for the final column. The second parameter is the reporting
-## @description subsystem (or test) that is providing the vote. The second parameter
-## @description is always required. The third parameter is any extra verbage that goes
-## @description with that subsystem.
-## @audience public
-## @stability stable
-## @replaceable no
-## @param +1/0/-1/null
-## @param subsystem
-## @param string
-## @return Elapsed time display
-function add_jira_table
-{
- local value=$1
- local subsystem=$2
- shift 2
-
- local color
- local calctime=0
-
- local -r elapsed=$(stop_clock)
-
- if [[ ${elapsed} -lt 0 ]]; then
- calctime="N/A"
- else
- printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
- fi
-
- echo ""
- echo "Elapsed time: ${calctime}"
- echo ""
-
- case ${value} in
- 1|+1)
- value="+1"
- color="green"
- ;;
- -1)
- color="red"
- ;;
- 0)
- color="blue"
- ;;
- null)
- ;;
- esac
-
- if [[ -z ${color} ]]; then
- JIRA_COMMENT_TABLE[${JTC}]="| | ${subsystem} | | ${*:-} |"
- JTC=$(( JTC+1 ))
- else
- JIRA_COMMENT_TABLE[${JTC}]="| {color:${color}}${value}{color} | ${subsystem} | ${calctime} | $* |"
- JTC=$(( JTC+1 ))
- fi
-}
-
-## @description Put the final environment information at the bottom
-## @description of the footer table
-## @stability stable
-## @audience private
-## @replaceable yes
-function close_jira_footer
-{
- # shellcheck disable=SC2016
- local -r javaversion=$("${JAVA_HOME}/bin/java" -version 2>&1 | head -1 | ${AWK} '{print $NF}' | tr -d \")
- local -r unamea=$(uname -a)
-
- add_jira_footer "Java" "${javaversion}"
- add_jira_footer "uname" "${unamea}"
-}
-
-## @description Put the final elapsed time at the bottom of the table.
-## @audience private
-## @stability stable
-## @replaceable no
-function close_jira_table
-{
-
- local -r elapsed=$(stop_global_clock)
-
- if [[ ${elapsed} -lt 0 ]]; then
- calctime="N/A"
- else
- printf -v calctime "%3sm %02ss" $((elapsed/60)) $((elapsed%60))
- fi
-
- echo ""
- echo "Total Elapsed time: ${calctime}"
- echo ""
-
-
- JIRA_COMMENT_TABLE[${JTC}]="| | | ${calctime} | |"
- JTC=$(( JTC+1 ))
-}
-
-## @description Add to the footer of the display. @@BASE@@ will get replaced with the
-## @description correct location for the local filesystem in dev mode or the URL for
-## @description Jenkins mode.
-## @audience public
-## @stability stable
-## @replaceable no
-## @param subsystem
-## @param string
-function add_jira_footer
-{
- local subsystem=$1
- shift 1
-
- JIRA_FOOTER_TABLE[${JFC}]="| ${subsystem} | $* |"
- JFC=$(( JFC+1 ))
-}
-
-## @description Special table just for unit test failures
-## @audience public
-## @stability stable
-## @replaceable no
-## @param failurereason
-## @param testlist
-function add_jira_test_table
-{
- local failure=$1
- shift 1
-
- JIRA_TEST_TABLE[${JTT}]="| ${failure} | $* |"
- JTT=$(( JTT+1 ))
-}
-
-## @description Large display for the user console
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-## @return large chunk of text
-function big_console_header
-{
- local text="$*"
- local spacing=$(( (75+${#text}) /2 ))
- printf "\n\n"
- echo "============================================================================"
- echo "============================================================================"
- printf "%*s\n" ${spacing} "${text}"
- echo "============================================================================"
- echo "============================================================================"
- printf "\n\n"
-}
-
-## @description Remove {color} tags from a string
-## @audience public
-## @stability stable
-## @replaceable no
-## @param string
-## @return string
-function colorstripper
-{
- local string=$1
- shift 1
-
- local green=""
- local white=""
- local red=""
- local blue=""
-
- echo "${string}" | \
- ${SED} -e "s,{color:red},${red},g" \
- -e "s,{color:green},${green},g" \
- -e "s,{color:blue},${blue},g" \
- -e "s,{color},${white},g"
-}
-
-## @description Find the largest size of a column of an array
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return size
-function findlargest
-{
- local column=$1
- shift
- local a=("$@")
- local sizeofa=${#a[@]}
- local i=0
-
- until [[ ${i} -gt ${sizeofa} ]]; do
- # shellcheck disable=SC2086
- string=$( echo ${a[$i]} | cut -f$((column + 1)) -d\| )
- if [[ ${#string} -gt $maxlen ]]; then
- maxlen=${#string}
- fi
- i=$((i+1))
- done
- echo "${maxlen}"
-}
-
-## @description Verify that ${JAVA_HOME} is defined
-## @audience public
-## @stability stable
-## @replaceable no
-## @return 1 - no JAVA_HOME
-## @return 0 - JAVA_HOME defined
-function find_java_home
-{
- start_clock
- if [[ -z ${JAVA_HOME:-} ]]; then
- case $(uname -s) in
- Darwin)
- if [[ -z "${JAVA_HOME}" ]]; then
- if [[ -x /usr/libexec/java_home ]]; then
- JAVA_HOME="$(/usr/libexec/java_home)"
- export JAVA_HOME
- else
- export JAVA_HOME=/Library/Java/Home
- fi
- fi
- ;;
- *)
- ;;
- esac
- fi
-
- if [[ -z ${JAVA_HOME:-} ]]; then
- echo "JAVA_HOME is not defined."
- add_jira_table -1 pre-patch "JAVA_HOME is not defined."
- return 1
- fi
- return 0
-}
-
-## @description Write the contents of a file to jenkins
-## @params filename
-## @stability stable
-## @audience public
-## @returns ${JIRACLI} exit code
-function write_to_jira
-{
- local -r commentfile=${1}
- shift
-
- local retval
-
- if [[ ${OFFLINE} == false
- && ${JENKINS} == true ]]; then
- export USER=hudson
- # shellcheck disable=SC2086
- ${JIRACLI} --comment "$(cat ${commentfile})" \
- -s https://issues.apache.org/jira \
- -a addcomment -u hadoopqa \
- -p "${JIRA_PASSWD}" \
- --issue "${ISSUE}"
- retval=$?
- ${JIRACLI} -s https://issues.apache.org/jira \
- -a logout -u hadoopqa \
- -p "${JIRA_PASSWD}"
- fi
- return ${retval}
-}
-
-## @description Verify that the patch directory is still in working order
-## @description since bad actors on some systems wipe it out. If not,
-## @description recreate it and then exit
-## @audience private
-## @stability evolving
-## @replaceable yes
-## @returns may exit on failure
-function verify_patchdir_still_exists
-{
- local -r commentfile=/tmp/testpatch.$$.${RANDOM}
- local extra=""
-
- if [[ ! -d ${PATCH_DIR} ]]; then
- rm "${commentfile}" 2>/dev/null
-
- echo "(!) The patch artifact directory has been removed! " > "${commentfile}"
- echo "This is a fatal error for test-patch.sh. Aborting. " >> "${commentfile}"
- echo
- cat ${commentfile}
- echo
- if [[ ${JENKINS} == true ]]; then
- if [[ -n ${NODE_NAME} ]]; then
- extra=" (node ${NODE_NAME})"
- fi
- echo "Jenkins${extra} information at ${BUILD_URL} may provide some hints. " >> "${commentfile}"
-
- write_to_jira ${commentfile}
- fi
-
- rm "${commentfile}"
- cleanup_and_exit ${RESULT}
- fi
-}
-
-## @description generate a list of all files and line numbers that
-## @description that were added/changed in the source repo
-## @audience private
-## @stability stable
-## @params filename
-## @replaceable no
-function compute_gitdiff
-{
- local outfile=$1
- local file
- local line
- local startline
- local counter
- local numlines
- local actual
-
- pushd "${BASEDIR}" >/dev/null
- while read line; do
- if [[ ${line} =~ ^\+\+\+ ]]; then
- file="./"$(echo "${line}" | cut -f2- -d/)
- continue
- elif [[ ${line} =~ ^@@ ]]; then
- startline=$(echo "${line}" | cut -f3 -d' ' | cut -f1 -d, | tr -d + )
- numlines=$(echo "${line}" | cut -f3 -d' ' | cut -s -f2 -d, )
- # if this is empty, then just this line
- # if it is 0, then no lines were added and this part of the patch
- # is strictly a delete
- if [[ ${numlines} == 0 ]]; then
- continue
- elif [[ -z ${numlines} ]]; then
- numlines=1
- fi
- counter=0
- until [[ ${counter} -gt ${numlines} ]]; do
- ((actual=counter+startline))
- echo "${file}:${actual}:" >> "${outfile}"
- ((counter=counter+1))
- done
- fi
- done < <("${GIT}" diff --unified=0 --no-color)
- popd >/dev/null
-}
-
-## @description Print the command to be executing to the screen. Then
-## @description run the command, sending stdout and stderr to the given filename
-## @description This will also ensure that any directories in ${BASEDIR} have
-## @description the exec bit set as a pre-exec step.
-## @audience public
-## @stability stable
-## @param filename
-## @param command
-## @param [..]
-## @replaceable no
-## @returns $?
-function echo_and_redirect
-{
- local logfile=$1
- shift
-
- verify_patchdir_still_exists
-
- find "${BASEDIR}" -type d -exec chmod +x {} \;
- echo "${*} > ${logfile} 2>&1"
- "${@}" > "${logfile}" 2>&1
-}
-
-## @description is PATCH_DIR relative to BASEDIR?
-## @audience public
-## @stability stable
-## @replaceable yes
-## @returns 1 - no, PATCH_DIR
-## @returns 0 - yes, PATCH_DIR - BASEDIR
-function relative_patchdir
-{
- local p=${PATCH_DIR#${BASEDIR}}
-
- if [[ ${#p} -eq ${#PATCH_DIR} ]]; then
- echo ${p}
- return 1
- fi
- p=${p#/}
- echo ${p}
- return 0
-}
-
-
-## @description Print the usage information
-## @audience public
-## @stability stable
-## @replaceable no
-function hadoop_usage
-{
- local -r up=$(echo ${PROJECT_NAME} | tr '[:lower:]' '[:upper:]')
-
- echo "Usage: test-patch.sh [options] patch-file | issue-number | http"
- echo
- echo "Where:"
- echo " patch-file is a local patch file containing the changes to test"
- echo " issue-number is a 'Patch Available' JIRA defect number (e.g. '${up}-9902') to test"
- echo " http is an HTTP address to download the patch file"
- echo
- echo "Options:"
- echo "--basedir=<dir> The directory to apply the patch to (default current directory)"
- echo "--branch=<ref> Forcibly set the branch"
- echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'trunk')"
- echo "--build-native=<bool> If true, then build native components (default 'true')"
- echo "--contrib-guide=<url> URL to point new users towards project conventions. (default Hadoop's wiki)"
- echo "--debug If set, then output some extra stuff to stderr"
- echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
- echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
- echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
- echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default '^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$')"
- echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
- echo "--offline Avoid connecting to the Internet"
- echo "--patch-dir=<dir> The directory for working and output files (default '/tmp/${PROJECT_NAME}-test-patch/pid')"
- echo "--plugins=<dir> A directory of user provided plugins. see test-patch.d for examples (default empty)"
- echo "--project=<name> The short name for project currently using test-patch (default 'hadoop')"
- echo "--resetrepo Forcibly clean the repo"
- echo "--run-tests Run all relevant tests below the base directory"
- echo "--skip-system-plugins Do not load plugins from ${BINDIR}/test-patch.d"
- echo "--testlist=<list> Specify which subsystem tests to use (comma delimited)"
- echo "--test-parallel=<bool> Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)"
- echo "--test-threads=<int> Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)"
-
- echo "Shell binary overrides:"
- echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
- echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')"
- echo "--file-cmd=<cmd> The 'file' command to use (default 'file')"
- echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
- echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
- echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
- echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
- echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')"
- echo "--sed-cmd=<cmd> The 'sed' command to use (default 'sed')"
-
- echo
- echo "Jenkins-only options:"
- echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
- echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
- echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
- echo "--jira-password=<pw> The password for the 'jira' command"
- echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup."
- echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
-}
-
-## @description Interpret the command line parameters
-## @audience private
-## @stability stable
-## @replaceable no
-## @params $@
-## @return May exit on failure
-function parse_args
-{
- local i
- local j
-
- for i in "$@"; do
- case ${i} in
- --awk-cmd=*)
- AWK=${i#*=}
- ;;
- --basedir=*)
- BASEDIR=${i#*=}
- ;;
- --branch=*)
- PATCH_BRANCH=${i#*=}
- ;;
- --branch-default=*)
- PATCH_BRANCH_DEFAULT=${i#*=}
- ;;
- --build-native=*)
- BUILD_NATIVE=${i#*=}
- ;;
- --contrib-guide=*)
- HOW_TO_CONTRIBUTE=${i#*=}
- ;;
- --debug)
- HADOOP_SHELL_SCRIPT_DEBUG=true
- ;;
- --diff-cmd=*)
- DIFF=${i#*=}
- ;;
- --dirty-workspace)
- DIRTY_WORKSPACE=true
- ;;
- --eclipse-home=*)
- ECLIPSE_HOME=${i#*=}
- ;;
- --file-cmd=*)
- FILE=${i#*=}
- ;;
- --findbugs-home=*)
- FINDBUGS_HOME=${i#*=}
- ;;
- --findbugs-strict-precheck)
- FINDBUGS_WARNINGS_FAIL_PRECHECK=true
- ;;
- --git-cmd=*)
- GIT=${i#*=}
- ;;
- --grep-cmd=*)
- GREP=${i#*=}
- ;;
- --help|-help|-h|help|--h|--\?|-\?|\?)
- hadoop_usage
- exit 0
- ;;
- --issue-re=*)
- ISSUE_RE=${i#*=}
- ;;
- --java-home=*)
- JAVA_HOME=${i#*=}
- ;;
- --jenkins)
- JENKINS=true
- TEST_PARALLEL=${TEST_PARALLEL:-true}
- ;;
- --jira-cmd=*)
- JIRACLI=${i#*=}
- ;;
- --jira-password=*)
- JIRA_PASSWD=${i#*=}
- ;;
- --modulelist=*)
- USER_MODULE_LIST=${i#*=}
- USER_MODULE_LIST=${USER_MODULE_LIST//,/ }
- hadoop_debug "Manually forcing modules ${USER_MODULE_LIST}"
- ;;
- --mvn-cmd=*)
- MVN=${i#*=}
- ;;
- --mv-patch-dir)
- RELOCATE_PATCH_DIR=true;
- ;;
- --offline)
- OFFLINE=true
- ;;
- --patch-cmd=*)
- PATCH=${i#*=}
- ;;
- --patch-dir=*)
- USER_PATCH_DIR=${i#*=}
- ;;
- --plugins=*)
- USER_PLUGIN_DIR=${i#*=}
- ;;
- --project=*)
- PROJECT_NAME=${i#*=}
- ;;
- --ps-cmd=*)
- PS=${i#*=}
- ;;
- --reexec)
- REEXECED=true
- start_clock
- add_jira_table 0 reexec "dev-support patch detected."
- ;;
- --resetrepo)
- RESETREPO=true
- ;;
- --run-tests)
- RUN_TESTS=true
- ;;
- --skip-system-plugins)
- LOAD_SYSTEM_PLUGINS=false
- ;;
- --testlist=*)
- testlist=${i#*=}
- testlist=${testlist//,/ }
- for j in ${testlist}; do
- hadoop_debug "Manually adding patch test subsystem ${j}"
- add_test "${j}"
- done
- ;;
- --test-parallel=*)
- TEST_PARALLEL=${i#*=}
- ;;
- --test-threads=*)
- TEST_THREADS=${i#*=}
- ;;
- --wget-cmd=*)
- WGET=${i#*=}
- ;;
- *)
- PATCH_OR_ISSUE=${i}
- ;;
- esac
- done
-
- # if we requested offline, pass that to mvn
- if [[ ${OFFLINE} == "true" ]] ; then
- MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
- fi
-
- # we need absolute dir for ${BASEDIR}
- cd "${CWD}"
- BASEDIR=$(cd -P -- "${BASEDIR}" >/dev/null && pwd -P)
-
- if [[ ${BUILD_NATIVE} == "true" ]] ; then
- NATIVE_PROFILE=-Pnative
- REQUIRE_TEST_LIB_HADOOP=-Drequire.test.libhadoop
- fi
- if [[ -z "${PATCH_OR_ISSUE}" ]]; then
- hadoop_usage
- exit 1
- fi
- if [[ ${JENKINS} == "true" ]] ; then
- echo "Running in Jenkins mode"
- ISSUE=${PATCH_OR_ISSUE}
- RESETREPO=true
- # shellcheck disable=SC2034
- ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}"
- else
- if [[ ${RESETREPO} == "true" ]] ; then
- echo "Running in destructive (--resetrepo) developer mode"
- else
- echo "Running in developer mode"
- fi
- JENKINS=false
- fi
-
- if [[ -n ${USER_PATCH_DIR} ]]; then
- PATCH_DIR="${USER_PATCH_DIR}"
- else
- PATCH_DIR=/tmp/${PROJECT_NAME}-test-patch/$$
- fi
-
- cd "${CWD}"
- if [[ ! -d ${PATCH_DIR} ]]; then
- mkdir -p "${PATCH_DIR}"
- if [[ $? == 0 ]] ; then
- echo "${PATCH_DIR} has been created"
- else
- echo "Unable to create ${PATCH_DIR}"
- cleanup_and_exit 1
- fi
- fi
-
- # we need absolute dir for PATCH_DIR
- PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
-
- GITDIFFLINES=${PATCH_DIR}/gitdifflines.txt
-
- if [[ ${TEST_PARALLEL} == "true" ]] ; then
- PARALLEL_TESTS_PROFILE=-Pparallel-tests
- if [[ -n ${TEST_THREADS:-} ]]; then
- TESTS_THREAD_COUNT="-DtestsThreadCount=$TEST_THREADS"
- fi
- fi
-}
-
-## @description Locate the pom.xml file for a given directory
-## @audience private
-## @stability stable
-## @replaceable no
-## @return directory containing the pom.xml
-function find_pom_dir
-{
- local dir
-
- dir=$(dirname "$1")
-
- hadoop_debug "Find pom dir for: ${dir}"
-
- while builtin true; do
- if [[ -f "${dir}/pom.xml" ]];then
- echo "${dir}"
- hadoop_debug "Found: ${dir}"
- return
- else
- dir=$(dirname "${dir}")
- fi
- done
-}
-
-## @description List of files that ${PATCH_DIR}/patch modifies
-## @audience private
-## @stability stable
-## @replaceable no
-## @return None; sets ${CHANGED_FILES}
-function find_changed_files
-{
- # get a list of all of the files that have been changed,
- # except for /dev/null (which would be present for new files).
- # Additionally, remove any a/ b/ patterns at the front
- # of the patch filenames and any revision info at the end
- # shellcheck disable=SC2016
- CHANGED_FILES=$(${GREP} -E '^(\+\+\+|---) ' "${PATCH_DIR}/patch" \
- | ${SED} \
- -e 's,^....,,' \
- -e 's,^[ab]/,,' \
- | ${GREP} -v /dev/null \
- | ${AWK} '{print $1}' \
- | sort -u)
-}
-
-## @description Find the modules of the maven build that ${PATCH_DIR}/patch modifies
-## @audience private
-## @stability stable
-## @replaceable no
-## @return None; sets ${CHANGED_MODULES}
-function find_changed_modules
-{
- # Come up with a list of changed files into ${TMP}
- local pomdirs
- local module
- local pommods
-
- # Now find all the modules that were changed
- for file in ${CHANGED_FILES}; do
- #shellcheck disable=SC2086
- pomdirs="${pomdirs} $(find_pom_dir ${file})"
- done
-
- # Filter out modules without code
- for module in ${pomdirs}; do
- ${GREP} "<packaging>pom</packaging>" "${module}/pom.xml" > /dev/null
- if [[ "$?" != 0 ]]; then
- pommods="${pommods} ${module}"
- fi
- done
-
- #shellcheck disable=SC2086
- CHANGED_MODULES=$(echo ${pommods} ${USER_MODULE_LIST} | tr ' ' '\n' | sort -u)
-}
-
-## @description git checkout the appropriate branch to test. Additionally, this calls
-## @description 'determine_issue' and 'determine_branch' based upon the context provided
-## @description in ${PATCH_DIR} and in git after checkout.
-## @audience private
-## @stability stable
-## @replaceable no
-## @return 0 on success. May exit on failure.
-function git_checkout
-{
- local currentbranch
- local exemptdir
-
- big_console_header "Confirming git environment"
-
- cd "${BASEDIR}"
- if [[ ! -d .git ]]; then
- hadoop_error "ERROR: ${BASEDIR} is not a git repo."
- cleanup_and_exit 1
- fi
-
- if [[ ${RESETREPO} == "true" ]] ; then
- ${GIT} reset --hard
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git reset is failing"
- cleanup_and_exit 1
- fi
-
- # if PATCH_DIR is in BASEDIR, then we don't want
- # git wiping it out.
- exemptdir=$(relative_patchdir)
- if [[ $? == 1 ]]; then
- ${GIT} clean -xdf
- else
- # we do, however, want it emptied of all _files_.
- # we need to leave _directories_ in case we are in
- # re-exec mode (which places a directory full of stuff in it)
- hadoop_debug "Exempting ${exemptdir} from clean"
- rm "${PATCH_DIR}/*" 2>/dev/null
- ${GIT} clean -xdf -e "${exemptdir}"
- fi
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git clean is failing"
- cleanup_and_exit 1
- fi
-
- ${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}"
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing"
- cleanup_and_exit 1
- fi
-
- determine_branch
- if [[ ${PATCH_BRANCH} =~ ^git ]]; then
- PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
- fi
-
- # we need to explicitly fetch in case the
- # git ref hasn't been brought in tree yet
- if [[ ${OFFLINE} == false ]]; then
-
- if [[ -f .git/rebase-apply ]]; then
- hadoop_error "ERROR: previous rebase failed. Aborting it."
- ${GIT} rebase --abort
- fi
-
- ${GIT} pull --rebase
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git pull is failing"
- cleanup_and_exit 1
- fi
- fi
- # forcibly checkout this branch or git ref
- ${GIT} checkout --force "${PATCH_BRANCH}"
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
- cleanup_and_exit 1
- fi
-
- # if we've selected a feature branch that has new changes
- # since our last build, we'll need to rebase to see those changes.
- if [[ ${OFFLINE} == false ]]; then
- ${GIT} pull --rebase
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: git pull is failing"
- cleanup_and_exit 1
- fi
- fi
-
- else
-
- status=$(${GIT} status --porcelain)
- if [[ "${status}" != "" && -z ${DIRTY_WORKSPACE} ]] ; then
- hadoop_error "ERROR: --dirty-workspace option not provided."
- hadoop_error "ERROR: can't run in a workspace that contains the following modifications"
- hadoop_error "${status}"
- cleanup_and_exit 1
- fi
-
- determine_branch
- if [[ ${PATCH_BRANCH} =~ ^git ]]; then
- PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
- fi
-
- currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD)
- if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then
- echo "WARNING: Current git branch is ${currentbranch} but patch is built for ${PATCH_BRANCH}."
- echo "WARNING: Continuing anyway..."
- PATCH_BRANCH=${currentbranch}
- fi
- fi
-
- determine_issue
-
- GIT_REVISION=$(${GIT} rev-parse --verify --short HEAD)
- # shellcheck disable=SC2034
- VERSION=${GIT_REVISION}_${ISSUE}_PATCH-${patchNum}
-
- if [[ "${ISSUE}" == 'Unknown' ]]; then
- echo "Testing patch on ${PATCH_BRANCH}."
- else
- echo "Testing ${ISSUE} patch on ${PATCH_BRANCH}."
- fi
-
- add_jira_footer "git revision" "${PATCH_BRANCH} / ${GIT_REVISION}"
-
- if [[ ! -f ${BASEDIR}/pom.xml ]]; then
- hadoop_error "ERROR: This verison of test-patch.sh only supports Maven-based builds. Aborting."
- add_jira_table -1 pre-patch "Unsupported build system."
- output_to_jira 1
- cleanup_and_exit 1
- fi
- return 0
-}
-
-## @description Confirm the source environment is compilable
-## @audience private
-## @stability stable
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_without_patch
-{
- local -r mypwd=$(pwd)
-
- big_console_header "Pre-patch ${PATCH_BRANCH} Java verification"
-
- start_clock
-
- verify_needed_test javac
-
- if [[ $? == 1 ]]; then
- echo "Compiling ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
- if [[ $? != 0 ]] ; then
- echo "${PATCH_BRANCH} compilation is broken?"
- add_jira_table -1 pre-patch "${PATCH_BRANCH} compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need javac tests."
- fi
-
- verify_needed_test javadoc
-
- if [[ $? == 1 ]]; then
- echo "Javadoc'ing ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} javadoc compilation is broken?"
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} JavaDoc compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need javadoc tests."
- fi
-
- verify_needed_test site
-
- if [[ $? == 1 ]]; then
- echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}SiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} site compilation is broken?"
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} site compilation may be broken."
- return 1
- fi
- else
- echo "Patch does not appear to need site tests."
- fi
-
- precheck_findbugs
-
- if [[ $? != 0 ]] ; then
- return 1
- fi
-
- add_jira_table 0 pre-patch "Pre-patch ${PATCH_BRANCH} compilation is healthy."
- return 0
-}
-
-## @description Confirm the given branch is a member of the list of space
-## @description delimited branches or a git ref
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param branch
-## @param branchlist
-## @return 0 on success
-## @return 1 on failure
-function verify_valid_branch
-{
- local branches=$1
- local check=$2
- local i
-
- # shortcut some common
- # non-resolvable names
- if [[ -z ${check} ]]; then
- return 1
- fi
-
- if [[ ${check} == patch ]]; then
- return 1
- fi
-
- if [[ ${check} =~ ^git ]]; then
- ref=$(echo "${check}" | cut -f2 -dt)
- count=$(echo "${ref}" | wc -c | tr -d ' ')
-
- if [[ ${count} == 8 || ${count} == 41 ]]; then
- return 0
- fi
- return 1
- fi
-
- for i in ${branches}; do
- if [[ "${i}" == "${check}" ]]; then
- return 0
- fi
- done
- return 1
-}
-
-## @description Try to guess the branch being tested using a variety of heuristics
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success, with PATCH_BRANCH updated appropriately
-## @return 1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT
-function determine_branch
-{
- local allbranches
- local patchnamechunk
-
- hadoop_debug "Determine branch"
-
- # something has already set this, so move on
- if [[ -n ${PATCH_BRANCH} ]]; then
- return
- fi
-
- pushd "${BASEDIR}" > /dev/null
-
- # developer mode, existing checkout, whatever
- if [[ "${DIRTY_WORKSPACE}" == true ]];then
- PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD)
- echo "dirty workspace mode; applying against existing branch"
- return
- fi
-
- allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g)
-
- for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
- hadoop_debug "Determine branch: starting with ${j}"
- # shellcheck disable=SC2016
- patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}')
-
- # ISSUE.branch.##.patch
- hadoop_debug "Determine branch: ISSUE.branch.##.patch"
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. )
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
-
- # ISSUE-branch-##.patch
- hadoop_debug "Determine branch: ISSUE-branch-##.patch"
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
-
- # ISSUE-##.patch.branch
- hadoop_debug "Determine branch: ISSUE-##.patch.branch"
- # shellcheck disable=SC2016
- PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}')
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
-
- # ISSUE-branch.##.patch
- hadoop_debug "Determine branch: ISSUE-branch.##.patch"
- # shellcheck disable=SC2016
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null)
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
- done
-
- PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}"
-
- popd >/dev/null
-}
-
-## @description Try to guess the issue being tested using a variety of heuristics
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success, with ISSUE updated appropriately
-## @return 1 on failure, with ISSUE updated to "Unknown"
-function determine_issue
-{
- local patchnamechunk
- local maybeissue
-
- hadoop_debug "Determine issue"
-
- # we can shortcut jenkins
- if [[ ${JENKINS} == true ]]; then
- ISSUE=${PATCH_OR_ISSUE}
- return 0
- fi
-
- # shellcheck disable=SC2016
- patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}')
-
- maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-)
-
- if [[ ${maybeissue} =~ ${ISSUE_RE} ]]; then
- ISSUE=${maybeissue}
- return 0
- fi
-
- ISSUE="Unknown"
- return 1
-}
-
-## @description Add the given test type
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-function add_test
-{
- local testname=$1
-
- hadoop_debug "Testing against ${testname}"
-
- if [[ -z ${NEEDED_TESTS} ]]; then
- hadoop_debug "Setting tests to ${testname}"
- NEEDED_TESTS=${testname}
- elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then
- hadoop_debug "Adding ${testname}"
- NEEDED_TESTS="${NEEDED_TESTS} ${testname}"
- fi
-}
-
-## @description Verify if a given test was requested
-## @audience public
-## @stability stable
-## @replaceable yes
-## @param test
-## @return 1 = yes
-## @return 0 = no
-function verify_needed_test
-{
- local i=$1
-
- if [[ ${NEEDED_TESTS} =~ $i ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Use some heuristics to determine which long running
-## @description tests to run
-## @audience private
-## @stability stable
-## @replaceable no
-function determine_needed_tests
-{
- local i
-
- for i in ${CHANGED_FILES}; do
- if [[ ${i} =~ src/main/webapp ]]; then
- hadoop_debug "tests/webapp: ${i}"
- elif [[ ${i} =~ \.sh
- || ${i} =~ \.cmd
- ]]; then
- hadoop_debug "tests/shell: ${i}"
- elif [[ ${i} =~ \.md$
- || ${i} =~ \.md\.vm$
- || ${i} =~ src/site
- || ${i} =~ src/main/docs
- ]]; then
- hadoop_debug "tests/site: ${i}"
- add_test site
- elif [[ ${i} =~ \.c$
- || ${i} =~ \.cc$
- || ${i} =~ \.h$
- || ${i} =~ \.hh$
- || ${i} =~ \.proto$
- || ${i} =~ src/test
- || ${i} =~ \.cmake$
- || ${i} =~ CMakeLists.txt
- ]]; then
- hadoop_debug "tests/units: ${i}"
- add_test javac
- add_test unit
- elif [[ ${i} =~ pom.xml$
- || ${i} =~ \.java$
- || ${i} =~ src/main
- ]]; then
- hadoop_debug "tests/javadoc+units: ${i}"
- add_test javadoc
- add_test javac
- add_test unit
- fi
-
- if [[ ${i} =~ \.java$ ]]; then
- add_test findbugs
- fi
-
- for plugin in ${PLUGINS}; do
- if declare -f ${plugin}_filefilter >/dev/null 2>&1; then
- "${plugin}_filefilter" "${i}"
- fi
- done
- done
-
- add_jira_footer "Optional Tests" "${NEEDED_TESTS}"
-}
-
-## @description Given ${PATCH_ISSUE}, determine what type of patch file is in use, and do the
-## @description necessary work to place it into ${PATCH_DIR}/patch.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure, may exit
-function locate_patch
-{
- local notSureIfPatch=false
- hadoop_debug "locate patch"
-
- if [[ -f ${PATCH_OR_ISSUE} ]]; then
- PATCH_FILE="${PATCH_OR_ISSUE}"
- else
- if [[ ${PATCH_OR_ISSUE} =~ ^http ]]; then
- echo "Patch is being downloaded at $(date) from"
- PATCHURL="${PATCH_OR_ISSUE}"
- else
- ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
-
- if [[ $? != 0 ]];then
- hadoop_error "ERROR: Unable to determine what ${PATCH_OR_ISSUE} may reference."
- cleanup_and_exit 1
- fi
-
- if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then
- if [[ ${JENKINS} == true ]]; then
- hadoop_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
- cleanup_and_exit 1
- else
- hadoop_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
- fi
- fi
-
- relativePatchURL=$(${GREP} -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PATCH_DIR}/jira" | ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${GREP} -o '/jira/secure/attachment/[0-9]*/[^"]*')
- PATCHURL="http://issues.apache.org${relativePatchURL}"
- if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
- notSureIfPatch=true
- fi
- patchNum=$(echo "${PATCHURL}" | ${GREP} -o '[0-9]*/' | ${GREP} -o '[0-9]*')
- echo "${ISSUE} patch is being downloaded at $(date) from"
- fi
- echo "${PATCHURL}"
- add_jira_footer "Patch URL" "${PATCHURL}"
- ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
- if [[ $? != 0 ]];then
- hadoop_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
- cleanup_and_exit 1
- fi
- PATCH_FILE="${PATCH_DIR}/patch"
- fi
-
- if [[ ! -f "${PATCH_DIR}/patch" ]]; then
- cp "${PATCH_FILE}" "${PATCH_DIR}/patch"
- if [[ $? == 0 ]] ; then
- echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}"
- else
- hadoop_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}"
- cleanup_and_exit 1
- fi
- fi
- if [[ ${notSureIfPatch} == "true" ]]; then
- guess_patch_file "${PATCH_DIR}/patch"
- if [[ $? != 0 ]]; then
- hadoop_error "ERROR: ${PATCHURL} is not a patch file."
- cleanup_and_exit 1
- else
- hadoop_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry."
- add_jira_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
- fi
- fi
-}
-
-## @description Given a possible patch file, guess if it's a patch file without using smart-apply-patch
-## @audience private
-## @stability evolving
-## @param path to patch file to test
-## @return 0 we think it's a patch file
-## @return 1 we think it's not a patch file
-function guess_patch_file
-{
- local patch=$1
- local fileOutput
-
- hadoop_debug "Trying to guess is ${patch} is a patch file."
- fileOutput=$("${FILE}" "${patch}")
- if [[ $fileOutput =~ \ diff\ ]]; then
- hadoop_debug "file magic says it's a diff."
- return 0
- fi
- fileOutput=$(head -n 1 "${patch}" | "${EGREP}" "^(From [a-z0-9]* Mon Sep 17 00:00:00 2001)|(diff .*)|(Index: .*)$")
- if [[ $? == 0 ]]; then
- hadoop_debug "first line looks like a patch file."
- return 0
- fi
- return 1
-}
-
-## @description Given ${PATCH_DIR}/patch, verify the patch is good using ${BINDIR}/smart-apply-patch.sh
-## @description in dryrun mode.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function verify_patch_file
-{
- # Before building, check to make sure that the patch is valid
- export PATCH
-
- "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch" dryrun
- if [[ $? != 0 ]] ; then
- echo "PATCH APPLICATION FAILED"
- add_jira_table -1 patch "The patch command could not apply the patch during dryrun."
- return 1
- else
- return 0
- fi
-}
-
-## @description Given ${PATCH_DIR}/patch, apply the patch using ${BINDIR}/smart-apply-patch.sh
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return exit on failure
-function apply_patch_file
-{
- big_console_header "Applying patch"
-
- export PATCH
- "${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch"
- if [[ $? != 0 ]] ; then
- echo "PATCH APPLICATION FAILED"
- ((RESULT = RESULT + 1))
- add_jira_table -1 patch "The patch command could not apply the patch."
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
- return 0
-}
-
-
-## @description If this actually patches the files used for the QA process
-## @description under dev-support and its subdirectories, then
-## @description run with the patched version for the test.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return none; otherwise relaunches
-function check_reexec
-{
- local commentfile=${PATCH_DIR}/tp.${RANDOM}
-
- if [[ ${REEXECED} == true ]]; then
- big_console_header "Re-exec mode detected. Continuing."
- return
- fi
-
- if [[ ! ${CHANGED_FILES} =~ dev-support/test-patch
- && ! ${CHANGED_FILES} =~ dev-support/smart-apply ]] ; then
- return
- fi
-
- big_console_header "dev-support patch detected"
-
- if [[ ${RESETREPO} == false ]]; then
- ((RESULT = RESULT + 1))
- hadoop_debug "can't destructively change the working directory. run with '--resetrepo' please. :("
- add_jira_table -1 dev-support "Couldn't test dev-support changes because we aren't configured to destructively change the working directory."
- return
- fi
-
- printf "\n\nRe-executing against patched versions to test.\n\n"
-
- apply_patch_file
-
- if [[ ${JENKINS} == true ]]; then
-
- rm "${commentfile}" 2>/dev/null
-
- echo "(!) A patch to the files used for the QA process has been detected. " > "${commentfile}"
- echo "Re-executing against the patched versions to perform further tests. " >> "${commentfile}"
- echo "The console is at ${BUILD_URL}console in case of problems." >> "${commentfile}"
-
- write_to_jira "${commentfile}"
- rm "${commentfile}"
- fi
-
- cd "${CWD}"
- mkdir -p "${PATCH_DIR}/dev-support-test"
- (cd "${BINDIR}"; tar cpf - . ) \
- | (cd "${PATCH_DIR}/dev-support-test"; tar xpf - )
-
- big_console_header "exec'ing test-patch.sh now..."
-
- exec "${PATCH_DIR}/dev-support-test/test-patch.sh" \
- --reexec \
- --branch="${PATCH_BRANCH}" \
- --patch-dir="${PATCH_DIR}" \
- "${USER_PARAMS[@]}"
-}
-
-## @description Check the current directory for @author tags
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_author
-{
- local authorTags
-
- big_console_header "Checking there are no @author tags in the patch."
-
- start_clock
-
- if [[ ${CHANGED_FILES} =~ dev-support/test-patch ]]; then
- add_jira_table 0 @author "Skipping @author checks as test-patch has been patched."
- return 0
- fi
-
- authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch")
- echo "There appear to be ${authorTags} @author tags in the patch."
- if [[ ${authorTags} != 0 ]] ; then
- add_jira_table -1 @author \
- "The patch appears to contain ${authorTags} @author tags which the Hadoop" \
- " community has agreed to not allow in code contributions."
- return 1
- fi
- add_jira_table +1 @author "The patch does not contain any @author tags."
- return 0
-}
-
-## @description Check the patch file for changed/new tests
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_modified_unittests
-{
- local testReferences=0
- local i
-
- verify_needed_test unit
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "Checking there are new or changed tests in the patch."
-
- start_clock
-
- for i in ${CHANGED_FILES}; do
- if [[ ${i} =~ /test/ ]]; then
- ((testReferences=testReferences + 1))
- fi
- done
-
- echo "There appear to be ${testReferences} test file(s) referenced in the patch."
- if [[ ${testReferences} == 0 ]] ; then
- add_jira_table -1 "tests included" \
- "The patch doesn't appear to include any new or modified tests. " \
- "Please justify why no new tests are needed for this patch." \
- "Also please list what manual steps were performed to verify this patch."
- return 1
- fi
- add_jira_table +1 "tests included" \
- "The patch appears to include ${testReferences} new or modified test files."
- return 0
-}
-
-## @description Helper for check_javadoc
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function count_javadoc_warns
-{
- local warningfile=$1
-
- #shellcheck disable=SC2016,SC2046
- return $(${EGREP} "^[0-9]+ warnings$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}')
-}
-
-## @description Count and compare the number of JavaDoc warnings pre- and post- patch
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_javadoc
-{
- local numBranchJavadocWarnings
- local numPatchJavadocWarnings
-
- verify_needed_test javadoc
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need javadoc checks."
- return 0
- fi
-
- big_console_header "Determining number of patched javadoc warnings"
-
- start_clock
-
- if [[ -d hadoop-project ]]; then
- (cd hadoop-project; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
- fi
- if [[ -d hadoop-common-project/hadoop-annotations ]]; then
- (cd hadoop-common-project/hadoop-annotations; "${MVN}" "${MAVEN_ARGS[@]}" install > /dev/null 2>&1)
- fi
- echo_and_redirect "${PATCH_DIR}/patchJavadocWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess
- count_javadoc_warns "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt"
- numBranchJavadocWarnings=$?
- count_javadoc_warns "${PATCH_DIR}/patchJavadocWarnings.txt"
- numPatchJavadocWarnings=$?
-
- echo "There appear to be ${numBranchJavadocWarnings} javadoc warnings before the patch and ${numPatchJavadocWarnings} javadoc warnings after applying the patch."
- if [[ ${numBranchJavadocWarnings} != "" && ${numPatchJavadocWarnings} != "" ]] ; then
- if [[ ${numPatchJavadocWarnings} -gt ${numBranchJavadocWarnings} ]] ; then
-
- ${GREP} -i warning "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarnings.txt" > "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt"
- ${GREP} -i warning "${PATCH_DIR}/patchJavadocWarnings.txt" > "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
- ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" \
- "${PATCH_DIR}/patchJavadocWarningsFiltered.txt" \
- > "${PATCH_DIR}/diffJavadocWarnings.txt"
- rm -f "${PATCH_DIR}/${PATCH_BRANCH}JavadocWarningsFiltered.txt" "${PATCH_DIR}/patchJavadocWarningsFiltered.txt"
-
- add_jira_table -1 javadoc "The applied patch generated "\
- "$((numPatchJavadocWarnings-numBranchJavadocWarnings))" \
- " additional warning messages."
- add_jira_footer javadoc "@@BASE@@/diffJavadocWarnings.txt"
- return 1
- fi
- fi
- add_jira_table +1 javadoc "There were no new javadoc warning messages."
- return 0
-}
-
-## @description Make sure site still compiles
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_site
-{
- local -r mypwd=$(pwd)
-
- verify_needed_test site
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need site checks."
- return 0
- fi
-
- big_console_header "Determining if patched site still builds"
-
- start_clock
-
- echo "site creation for ${mypwd}"
- echo_and_redirect "${PATCH_DIR}/patchSiteWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean site site:stage -DskipTests -Dmaven.javadoc.skip=true -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- echo "Site compilation is broken"
- add_jira_table -1 site "Site compilation is broken."
- add_jira_footer site "@@BASE@@/patchSiteWarnings.txt"
- return 1
- fi
- add_jira_table +1 site "Site still builds."
- return 0
-}
-
-## @description Helper for check_javac
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function count_javac_warns
-{
- local warningfile=$1
- #shellcheck disable=SC2016,SC2046
- return $(${AWK} 'BEGIN {total = 0} {total += 1} END {print total}' "${warningfile}")
-}
-
-## @description Count and compare the number of javac warnings pre- and post- patch
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_javac
-{
- local branchJavacWarnings
- local patchJavacWarnings
-
- verify_needed_test javac
-
- if [[ $? == 0 ]]; then
- echo "This patch does not appear to need javac checks."
- return 0
- fi
-
- big_console_header "Determining number of patched javac warnings."
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchJavacWarnings.txt" "${MVN}" "${MAVEN_ARGS[@]}" clean test -DskipTests -D${PROJECT_NAME}PatchProcess ${NATIVE_PROFILE} -Ptest-patch
- if [[ $? != 0 ]] ; then
- add_jira_table -1 javac "The patch appears to cause the build to fail."
- return 2
- fi
- ### Compare ${PATCH_BRANCH} and patch javac warning numbers
- if [[ -f ${PATCH_DIR}/patchJavacWarnings.txt ]] ; then
- ${GREP} '\[WARNING\]' "${PATCH_DIR}/${PATCH_BRANCH}JavacWarnings.txt" > "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
- ${GREP} '\[WARNING\]' "${PATCH_DIR}/patchJavacWarnings.txt" > "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
-
- count_javac_warns "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt"
- branchJavacWarnings=$?
- count_javac_warns "${PATCH_DIR}/filteredPatchJavacWarnings.txt"
- patchJavacWarnings=$?
-
- echo "There appear to be ${branchJavacWarnings} javac compiler warnings before the patch and ${patchJavacWarnings} javac compiler warnings after applying the patch."
- if [[ ${patchJavacWarnings} != "" && ${branchJavacWarnings} != "" ]] ; then
- if [[ ${patchJavacWarnings} -gt ${branchJavacWarnings} ]] ; then
-
- ${DIFF} "${PATCH_DIR}/filtered${PATCH_BRANCH}JavacWarnings.txt" \
- "${PATCH_DIR}/filteredPatchJavacWarnings.txt" \
- > "${PATCH_DIR}/diffJavacWarnings.txt"
-
- add_jira_table -1 javac "The applied patch generated "\
- "$((patchJavacWarnings-branchJavacWarnings))" \
- " additional warning messages."
-
- add_jira_footer javac "@@BASE@@/diffJavacWarnings.txt"
-
- return 1
- fi
- fi
- fi
-
- add_jira_table +1 javac "There were no new javac warning messages."
- return 0
-}
-
-## @description Verify all files have an Apache License
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_apachelicense
-{
- big_console_header "Determining number of patched release audit warnings."
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchReleaseAuditOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" apache-rat:check -D${PROJECT_NAME}PatchProcess
- #shellcheck disable=SC2038
- find "${BASEDIR}" -name rat.txt | xargs cat > "${PATCH_DIR}/patchReleaseAuditWarnings.txt"
-
- ### Compare ${PATCH_BRANCH} and patch release audit warning numbers
- if [[ -f ${PATCH_DIR}/patchReleaseAuditWarnings.txt ]] ; then
- patchReleaseAuditWarnings=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt")
- echo ""
- echo ""
- echo "There appear to be ${patchReleaseAuditWarnings} release audit warnings after applying the patch."
- if [[ ${patchReleaseAuditWarnings} != "" ]] ; then
- if [[ ${patchReleaseAuditWarnings} -gt 0 ]] ; then
- add_jira_table -1 "release audit" "The applied patch generated ${patchReleaseAuditWarnings} release audit warnings."
-
- ${GREP} '\!?????' "${PATCH_DIR}/patchReleaseAuditWarnings.txt" \
- > "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
- echo "Lines that start with ????? in the release audit "\
- "report indicate files that do not have an Apache license header." \
- >> "${PATCH_DIR}/patchReleaseAuditProblems.txt"
-
- add_jira_footer "Release Audit" "@@BASE@@/patchReleaseAuditProblems.txt"
-
- return 1
- fi
- fi
- fi
- add_jira_table 1 "release audit" "The applied patch does not increase the total number of release audit warnings."
- return 0
-}
-
-## @description Verify mvn install works
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_mvn_install
-{
- local retval
-
- verify_needed_test javadoc
- retval=$?
-
- verify_needed_test javac
- ((retval = retval + $? ))
-
- if [[ ${retval} == 0 ]]; then
- echo "This patch does not appear to need mvn install checks."
- return 0
- fi
-
- big_console_header "Installing all of the jars"
-
- start_clock
- echo_and_redirect "${PATCH_DIR}/jarinstall.txt" "${MVN}" "${MAVEN_ARGS[@]}" install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess
- retval=$?
- if [[ ${retval} != 0 ]]; then
- add_jira_table -1 install "The patch causes mvn install to fail."
- else
- add_jira_table +1 install "mvn install still works."
- fi
- return ${retval}
-}
-
-## @description are the needed bits for findbugs present?
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 findbugs will work for our use
-## @return 1 findbugs is missing some component
-function findbugs_is_installed
-{
- if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
- printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
- add_jira_table -1 findbugs "Findbugs is not installed."
- return 1
- fi
- return 0
-}
-
-## @description Run the maven findbugs plugin and record found issues in a bug database
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function findbugs_mvnrunner
-{
- local name=$1
- local logfile=$2
- local warnings_file=$3
-
- echo_and_redirect "${logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean test findbugs:findbugs -DskipTests \
- "-D${PROJECT_NAME}PatchProcess" < /dev/null
- if [[ $? != 0 ]]; then
- return 1
- fi
- cp target/findbugsXml.xml "${warnings_file}.xml"
-
- "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${name}" \
- "${warnings_file}.xml" "${warnings_file}.xml"
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${warnings_file}.xml" \
- "${warnings_file}.html"
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- return 0
-}
-
-## @description Track pre-existing findbugs warnings
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_findbugs
-{
- local -r mypwd=$(pwd)
- local module_suffix
- local modules=${CHANGED_MODULES}
- local module
- local findbugs_version
- local rc=0
- local module_findbugs_warnings
- local findbugs_warnings=0
-
- verify_needed_test findbugs
-
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need findbugs tests."
- return 0
- fi
-
- echo "findbugs baseline for ${mypwd}"
-
- findbugs_is_installed
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running findbugs in ${module}"
- module_suffix=$(basename "${module}")
- findbugs_mvnrunner "${PATCH_BRANCH}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}"
- (( rc = rc + $? ))
-
- if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
- #shellcheck disable=SC2016
- module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
- "${PATCH_BRANCH}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}".xml \
- | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- findbugs_warnings=$((findbugs_warnings+module_findbugs_warnings))
-
- if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
- add_jira_footer "Pre-patch Findbugs warnings" "@@BASE@@/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.html"
- fi
- fi
- popd >/dev/null
- done
-
- #shellcheck disable=SC2016
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/${PATCH_BRANCH}FindBugsOutput${module_suffix}.txt")
-
- if [[ ${rc} -ne 0 ]]; then
- echo "Pre-patch ${PATCH_BRANCH} findbugs is broken?"
- add_jira_table -1 pre-patch "Findbugs (version ${findbugs_version}) appears to be broken on ${PATCH_BRANCH}."
- return 1
- fi
-
- if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" && \
- ${findbugs_warnings} -gt 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} findbugs has ${findbugs_warnings} warnings."
- add_jira_table -1 pre-patch "Pre-patch ${PATCH_BRANCH} has ${findbugs_warnings} extant Findbugs (version ${findbugs_version}) warnings."
- return 1
- fi
-
- return 0
-}
-
-## @description Verify patch does not trigger any findbugs warnings
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_findbugs
-{
- local rc=0
- local module
- local modules=${CHANGED_MODULES}
- local module_suffix
- local combined_xml
- local newBugs
- local new_findbugs_warnings
- local new_findbugs_fixed_warnings
- local findbugs_warnings=0
- local findbugs_fixed_warnings=0
- local line
- local firstpart
- local secondpart
- local findbugs_version
-
- verify_needed_test findbugs
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "Determining number of patched Findbugs warnings."
-
- start_clock
-
- findbugs_is_installed
- if [[ $? != 0 ]]; then
- return 1
- fi
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running findbugs in ${module}"
- module_suffix=$(basename "${module}")
-
- findbugs_mvnrunner patch \
- "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}"
-
- if [[ $? != 0 ]] ; then
- ((rc = rc +1))
- echo "Post-patch findbugs compilation is broken."
- add_jira_table -1 findbugs "Post-patch findbugs ${module} compilation is broken."
- continue
- fi
-
- combined_xml="$PATCH_DIR/combinedFindbugsWarnings${module_suffix}.xml"
- newBugs="${PATCH_DIR}/newPatchFindbugsWarnings${module_suffix}"
- "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
- -output "${combined_xml}" \
- "${PATCH_DIR}/${PATCH_BRANCH}FindbugsWarnings${module_suffix}.xml" \
- "${PATCH_DIR}/patchFindbugsWarnings${module_suffix}.xml"
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- #shellcheck disable=SC2016
- new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
- "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
- #shellcheck disable=SC2016
- new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
- "${combined_xml}" "${newBugs}.xml" | ${AWK} '{print $1}')
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
- findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
- findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
-
- "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newBugs}.xml" \
- "${newBugs}.html"
- if [[ $? != 0 ]]; then
- popd >/dev/null
- return 1
- fi
-
- if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
- populate_test_table FindBugs "module:${module_suffix}"
- while read line; do
- firstpart=$(echo "${line}" | cut -f2 -d:)
- secondpart=$(echo "${line}" | cut -f9- -d' ')
- add_jira_test_table "" "${firstpart}:${secondpart}"
- done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newBugs}.xml")
-
- add_jira_footer "Findbugs warnings" "@@BASE@@/newPatchFindbugsWarnings${module_suffix}.html"
- fi
-
- popd >/dev/null
- done
-
- #shellcheck disable=SC2016
- findbugs_version=$(${AWK} 'match($0, /findbugs-maven-plugin:[^:]*:findbugs/) { print substr($0, RSTART + 22, RLENGTH - 31); exit }' "${PATCH_DIR}/patchFindBugsOutput${module_suffix}.txt")
-
- if [[ ${findbugs_warnings} -gt 0 ]] ; then
- add_jira_table -1 findbugs "The patch appears to introduce ${findbugs_warnings} new Findbugs (version ${findbugs_version}) warnings."
- return 1
- fi
-
- if [[ ${findbugs_fixed_warnings} -gt 0 ]] ; then
- add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings, and fixes ${findbugs_fixed_warnings} pre-existing warnings."
- else
- add_jira_table +1 findbugs "The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
- fi
- return 0
-}
-
-## @description Make sure Maven's eclipse generation works.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_mvn_eclipse
-{
- big_console_header "Running mvn eclipse:eclipse."
-
- verify_needed_test javac
- if [[ $? == 0 ]]; then
- echo "Patch does not touch any java files. Skipping mvn eclipse:eclipse"
- return 0
- fi
-
- start_clock
-
- echo_and_redirect "${PATCH_DIR}/patchEclipseOutput.txt" "${MVN}" "${MAVEN_ARGS[@]}" eclipse:eclipse -D${PROJECT_NAME}PatchProcess
- if [[ $? != 0 ]] ; then
- add_jira_table -1 eclipse:eclipse "The patch failed to build with eclipse:eclipse."
- return 1
- fi
- add_jira_table +1 eclipse:eclipse "The patch built with eclipse:eclipse."
- return 0
-}
-
-## @description Utility to push many tests into the failure list
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param testdesc
-## @param testlist
-function populate_test_table
-{
- local reason=$1
- shift
- local first=""
- local i
-
- for i in "$@"; do
- if [[ -z "${first}" ]]; then
- add_jira_test_table "${reason}" "${i}"
- first="${reason}"
- else
- add_jira_test_table " " "${i}"
- fi
- done
-}
-
-## @description Run and verify the output of the appropriate unit tests
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_unittests
-{
- verify_needed_test unit
-
- if [[ $? == 0 ]]; then
- echo "Existing unit tests do not test patched files. Skipping."
- return 0
- fi
-
- big_console_header "Running unit tests"
-
- start_clock
-
- local failed_tests=""
- local modules=${CHANGED_MODULES}
- local building_common=0
- local hdfs_modules
- local ordered_modules=""
- local failed_test_builds=""
- local test_timeouts=""
- local test_logfile
- local test_build_result
- local module_test_timeouts=""
- local result
- local totalresult=0
- local module_prefix
-
- #
- # If we are building hadoop-hdfs-project, we must build the native component
- # of hadoop-common-project first. In order to accomplish this, we move the
- # hadoop-hdfs subprojects to the end of the list so that common will come
- # first.
- #
- # Of course, we may not be building hadoop-common at all-- in this case, we
- # explicitly insert a mvn compile -Pnative of common, to ensure that the
- # native libraries show up where we need them.
- #
-
- for module in ${modules}; do
- if [[ ${module} == hadoop-hdfs-project* ]]; then
- hdfs_modules="${hdfs_modules} ${module}"
- elif [[ ${module} == hadoop-common-project* ]]; then
- ordered_modules="${ordered_modules} ${module}"
- building_common=1
- else
- ordered_modules="${ordered_modules} ${module}"
- fi
- done
-
- if [[ -n "${hdfs_modules}" ]]; then
- ordered_modules="${ordered_modules} ${hdfs_modules}"
- if [[ ${building_common} -eq 0 ]]; then
- echo " Building hadoop-common with -Pnative in order to provide libhadoop.so to the hadoop-hdfs unit tests."
- echo_and_redirect "${PATCH_DIR}/testrun_native.txt" "${MVN}" "${MAVEN_ARGS[@]}" compile ${NATIVE_PROFILE} "-D${PROJECT_NAME}PatchProcess"
- if [[ $? != 0 ]]; then
- add_jira_table -1 "native" "Failed to build the native portion " \
- "of hadoop-common prior to running the unit tests in ${ordered_modules}"
- return 1
- else
- add_jira_table +1 "native" "Pre-build of native portion"
- fi
- fi
- fi
-
- for module in ${ordered_modules}; do
- result=0
- start_clock
- pushd "${module}" >/dev/null
- module_suffix=$(basename "${module}")
- module_prefix=$(echo "${module}" | cut -f2 -d- )
-
- test_logfile=${PATCH_DIR}/testrun_${module_suffix}.txt
- echo " Running tests in ${module_suffix}"
- # shellcheck disable=2086
- echo_and_redirect "${test_logfile}" "${MVN}" "${MAVEN_ARGS[@]}" clean install -fae ${NATIVE_PROFILE} ${REQUIRE_TEST_LIB_HADOOP} ${PARALLEL_TESTS_PROFILE} ${TESTS_THREAD_COUNT} -D${PROJECT_NAME}PatchProcess
- test_build_result=$?
-
- add_jira_footer "${module_suffix} test log" "@@BASE@@/testrun_${module_suffix}.txt"
-
- # shellcheck disable=2016
- module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${test_logfile}")
- if [[ -n "${module_test_timeouts}" ]] ; then
- test_timeouts="${test_timeouts} ${module_test_timeouts}"
- result=1
- fi
-
- #shellcheck disable=SC2026,SC2038,SC2016
- module_failed_tests=$(find . -name 'TEST*.xml'\
- | xargs "${GREP}" -l -E "<failure|<error"\
- | ${AWK} -F/ '{sub("TEST-org.apache.",""); sub(".xml",""); print $NF}')
-
- if [[ -n "${module_failed_tests}" ]] ; then
- failed_tests="${failed_tests} ${module_failed_tests}"
- result=1
- fi
- if [[ ${test_build_result} != 0 && -z "${module_failed_tests}" && -z "${module_test_timeouts}" ]] ; then
- failed_test_builds="${failed_test_builds} ${module_suffix}"
- result=1
- fi
- popd >/dev/null
-
- if [[ $result == 1 ]]; then
- add_jira_table -1 "${module_prefix} tests" "Tests failed in ${module_suffix}."
- else
- add_jira_table +1 "${module_prefix} tests" "Tests passed in ${module_suffix}."
- fi
-
- ((totalresult = totalresult + result))
- done
-
- if [[ -n "${failed_tests}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Failed unit tests" ${failed_tests}
- fi
- if [[ -n "${test_timeouts}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Timed out tests" ${test_timeouts}
- fi
- if [[ -n "${failed_test_builds}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "Failed build" ${failed_test_builds}
- fi
-
- if [[ ${JENKINS} == true ]]; then
- add_jira_footer "Test Results" "${BUILD_URL}testReport/"
- fi
-
- if [[ ${totalresult} -gt 0 ]]; then
- return 1
- else
- return 0
- fi
-}
-
-## @description Print out the finished details on the console
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-## @return 0 on success
-## @return 1 on failure
-function output_to_console
-{
- local result=$1
- shift
- local i
- local ourstring
- local vote
- local subs
- local ela
- local comment
- local commentfile1="${PATCH_DIR}/comment.1"
- local commentfile2="${PATCH_DIR}/comment.2"
- local normaltop
- local line
- local seccoladj=0
- local spcfx=${PATCH_DIR}/spcl.txt
-
- if [[ ${result} == 0 ]]; then
- if [[ ${JENKINS} == false ]]; then
- {
- printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
- printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
- printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
- printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
- printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
- printf "ICB8bXwgICB8bXwgIAo"
- } > "${spcfx}"
- fi
- printf "\n\n+1 overall\n\n"
- else
- if [[ ${JENKINS} == false ]]; then
- {
- printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
- printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
- printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
- printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
- printf "ICAgICAgICAgICAgICAgICAK"
- } > "${spcfx}"
- fi
- printf "\n\n-1 overall\n\n"
- fi
-
- if [[ -f ${spcfx} ]]; then
- if which base64 >/dev/null 2>&1; then
- base64 --decode "${spcfx}" 2>/dev/null
- elif which openssl >/dev/null 2>&1; then
- openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
- fi
- echo
- echo
- rm "${spcfx}"
- fi
-
- seccoladj=$(findlargest 2 "${JIRA_COMMENT_TABLE[@]}")
- if [[ ${seccoladj} -lt 10 ]]; then
- seccoladj=10
- fi
-
- seccoladj=$((seccoladj + 2 ))
- i=0
- until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
- printf "%s\n" "${JIRA_HEADER[${i}]}"
- ((i=i+1))
- done
-
- printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
- echo "============================================================================"
- i=0
- until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
- ourstring=$(echo "${JIRA_COMMENT_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- vote=$(colorstripper "${vote}")
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- ela=$(echo "${ourstring}" | cut -f4 -d\|)
- comment=$(echo "${ourstring}" | cut -f5 -d\|)
-
- echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
- normaltop=$(head -1 "${commentfile1}")
- ${SED} -e '1d' "${commentfile1}" > "${commentfile2}"
-
- printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
- "${subs}" "${ela}" "${normaltop}"
- while read line; do
- printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}"
- done < "${commentfile2}"
-
- ((i=i+1))
- rm "${commentfile2}" "${commentfile1}" 2>/dev/null
- done
-
- if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
- seccoladj=$(findlargest 1 "${JIRA_TEST_TABLE[@]}")
- printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
- i=0
- until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
- ourstring=$(echo "${JIRA_TEST_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
- ((i=i+1))
- done
- fi
-
- printf "\n\n|| Subsystem || Report/Notes ||\n"
- echo "============================================================================"
- i=0
-
- until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
- comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
- printf "%s\n" "${comment}"
- ((i=i+1))
- done
-}
-
-## @description Print out the finished details to the JIRA issue
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-function output_to_jira
-{
- local result=$1
- local i
- local commentfile=${PATCH_DIR}/commentfile
- local comment
-
- rm "${commentfile}" 2>/dev/null
-
- if [[ ${JENKINS} != "true" ]] ; then
- return 0
- fi
-
- big_console_header "Adding comment to JIRA"
-
- add_jira_footer "Console output" "${BUILD_URL}console"
-
- if [[ ${result} == 0 ]]; then
- add_jira_header "(/) *{color:green}+1 overall{color}*"
- else
- add_jira_header "(x) *{color:red}-1 overall{color}*"
- fi
-
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- i=0
- until [[ $i -eq ${#JIRA_HEADER[@]} ]]; do
- printf "%s\n" "${JIRA_HEADER[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}"
-
- i=0
- until [[ $i -eq ${#JIRA_COMMENT_TABLE[@]} ]]; do
- printf "%s\n" "${JIRA_COMMENT_TABLE[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
-
-
- if [[ ${#JIRA_TEST_TABLE[@]} -gt 0 ]]; then
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Reason || Tests ||" >> "${commentfile}"
- i=0
- until [[ $i -eq ${#JIRA_TEST_TABLE[@]} ]]; do
- printf "%s\n" "${JIRA_TEST_TABLE[${i}]}" >> "${commentfile}"
- ((i=i+1))
- done
- fi
-
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
-
- echo "|| Subsystem || Report/Notes ||" >> "${commentfile}"
- i=0
- until [[ $i -eq ${#JIRA_FOOTER_TABLE[@]} ]]; do
- comment=$(echo "${JIRA_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g")
- printf "%s\n" "${comment}" >> "${commentfile}"
- ((i=i+1))
- done
-
- printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}"
-
- write_to_jira "${commentfile}"
-}
-
-## @description Clean the filesystem as appropriate and then exit
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param runresult
-function cleanup_and_exit
-{
- local result=$1
-
- if [[ ${JENKINS} == "true" && ${RELOCATE_PATCH_DIR} == "true" && \
- -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then
- # if PATCH_DIR is already inside BASEDIR, then
- # there is no need to move it since we assume that
- # Jenkins or whatever already knows where it is at
- # since it told us to put it there!
- relative_patchdir >/dev/null
- if [[ $? == 1 ]]; then
- hadoop_debug "mv ${PATCH_DIR} ${BASEDIR}"
- mv "${PATCH_DIR}" "${BASEDIR}"
- fi
- fi
- big_console_header "Finished build."
-
- # shellcheck disable=SC2086
- exit ${result}
-}
-
-## @description Driver to execute _postcheckout routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function postcheckout
-{
- local routine
- local plugin
-
- for routine in find_java_home verify_patch_file
- do
- verify_patchdir_still_exists
-
- hadoop_debug "Running ${routine}"
- ${routine}
-
- (( RESULT = RESULT + $? ))
- if [[ ${RESULT} != 0 ]] ; then
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
-
- if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then
-
- hadoop_debug "Running ${plugin}_postcheckout"
- #shellcheck disable=SC2086
- ${plugin}_postcheckout
-
- (( RESULT = RESULT + $? ))
- if [[ ${RESULT} != 0 ]] ; then
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
- fi
- done
-}
-
-## @description Driver to execute _preapply routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function preapply
-{
- local routine
- local plugin
-
- for routine in precheck_without_patch check_author \
- check_modified_unittests
- do
- verify_patchdir_still_exists
-
- hadoop_debug "Running ${routine}"
- ${routine}
-
- (( RESULT = RESULT + $? ))
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
-
- if declare -f ${plugin}_preapply >/dev/null 2>&1; then
-
- hadoop_debug "Running ${plugin}_preapply"
- #shellcheck disable=SC2086
- ${plugin}_preapply
-
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Driver to execute _postapply routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function postapply
-{
- local routine
- local plugin
- local retval
-
- compute_gitdiff "${GITDIFFLINES}"
-
- check_javac
- retval=$?
- if [[ ${retval} -gt 1 ]] ; then
- output_to_console 1
- output_to_jira 1
- cleanup_and_exit 1
- fi
-
- ((RESULT = RESULT + retval))
-
- for routine in check_javadoc check_apachelicense check_site
- do
- verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
- $routine
-
- (( RESULT = RESULT + $? ))
-
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_postapply >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_postapply"
- #shellcheck disable=SC2086
- ${plugin}_postapply
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Driver to execute _postinstall routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function postinstall
-{
- local routine
- local plugin
-
- for routine in check_mvn_eclipse check_findbugs
- do
- verify_patchdir_still_exists
- hadoop_debug "Running ${routine}"
- ${routine}
- (( RESULT = RESULT + $? ))
- done
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_postinstall >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_postinstall"
- #shellcheck disable=SC2086
- ${plugin}_postinstall
- (( RESULT = RESULT + $? ))
- fi
- done
-
-}
-
-## @description Driver to execute _tests routines
-## @audience private
-## @stability evolving
-## @replaceable no
-function runtests
-{
- local plugin
-
- ### Run tests for Jenkins or if explictly asked for by a developer
- if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then
-
- verify_patchdir_still_exists
- check_unittests
-
- (( RESULT = RESULT + $? ))
- fi
-
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_tests >/dev/null 2>&1; then
- hadoop_debug "Running ${plugin}_tests"
- #shellcheck disable=SC2086
- ${plugin}_tests
- (( RESULT = RESULT + $? ))
- fi
- done
-}
-
-## @description Import content from test-patch.d and optionally
-## @description from user provided plugin directory
-## @audience private
-## @stability evolving
-## @replaceable no
-function importplugins
-{
- local i
- local files=()
-
- if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
- if [[ -d "${BINDIR}/test-patch.d" ]]; then
- files=(${BINDIR}/test-patch.d/*.sh)
- fi
- fi
-
- if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
- hadoop_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
- files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
- fi
-
- for i in "${files[@]}"; do
- hadoop_debug "Importing ${i}"
- . "${i}"
- done
-}
-
-## @description Register test-patch.d plugins
-## @audience public
-## @stability stable
-## @replaceable no
-function add_plugin
-{
- PLUGINS="${PLUGINS} $1"
-}
-
-###############################################################################
-###############################################################################
-###############################################################################
-
-big_console_header "Bootstrapping test harness"
-
-setup_defaults
-
-parse_args "$@"
-
-importplugins
-
-locate_patch
-
-find_changed_files
-
-determine_needed_tests
-
-# from here on out, we'll be in ${BASEDIR} for cwd
-# routines need to pushd/popd if they change.
-git_checkout
-RESULT=$?
-if [[ ${JENKINS} == "true" ]] ; then
- if [[ ${RESULT} != 0 ]] ; then
- exit 100
- fi
-fi
-
-check_reexec
-
-postcheckout
-
-find_changed_modules
-
-preapply
-
-apply_patch_file
-
-postapply
-
-check_mvn_install
-
-postinstall
-
-runtests
-
-close_jira_footer
-
-close_jira_table
-
-output_to_console ${RESULT}
-output_to_jira ${RESULT}
-cleanup_and_exit ${RESULT}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 108834b..68ad350 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -513,10 +513,9 @@
<goal>exec</goal>
</goals>
<configuration>
- <executable>python</executable>
+ <executable>${basedir}/../../dev-support/bin/shelldocs</executable>
<workingDirectory>src/site/markdown</workingDirectory>
<arguments>
- <argument>${basedir}/../../dev-support/shelldocs.py</argument>
<argument>--skipprnorep</argument>
<argument>--output</argument>
<argument>${basedir}/src/site/markdown/UnixShellAPI.md</argument>
@@ -982,11 +981,10 @@
<goal>exec</goal>
</goals>
<configuration>
- <executable>python</executable>
+ <executable>${basedir}/../../dev-support/bin/releasedocmaker</executable>
<workingDirectory>src/site/markdown/release/</workingDirectory>
<requiresOnline>true</requiresOnline>
<arguments>
- <argument>${basedir}/../../dev-support/releasedocmaker.py</argument>
<argument>--index</argument>
<argument>--license</argument>
<argument>--project</argument>
[2/2] hadoop git commit: HADOOP-12651. Replace dev-support with
wrappers to Yetus (aw)
Posted by aw...@apache.org.
HADOOP-12651. Replace dev-support with wrappers to Yetus (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cecad2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cecad2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cecad2d
Branch: refs/heads/trunk
Commit: 8cecad2d564d89d0f56e98de80a6f9b0bcb39d7a
Parents: c304890
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jan 21 12:22:03 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jan 21 12:22:03 2016 -0800
----------------------------------------------------------------------
BUILDING.txt | 2 +-
dev-support/releasedocmaker.py | 580 -----
dev-support/shelldocs.py | 271 ---
dev-support/smart-apply-patch.sh | 187 --
dev-support/test-patch.d/checkstyle.sh | 205 --
dev-support/test-patch.d/shellcheck.sh | 178 --
dev-support/test-patch.d/whitespace.sh | 46 -
dev-support/test-patch.sh | 2814 ----------------------
hadoop-common-project/hadoop-common/pom.xml | 6 +-
9 files changed, 3 insertions(+), 4286 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 6a098fa..3eb9e9c 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -39,7 +39,7 @@ and all required tools for testing and building have been installed and configur
Note that from within this docker environment you ONLY have access to the Hadoop source
tree from where you started. So if you need to run
- dev-support/test-patch.sh /path/to/my.patch
+ dev-support/bin/test-patch /path/to/my.patch
then the patch must be placed inside the hadoop source tree.
Known issues:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
deleted file mode 100755
index d2e5dda..0000000
--- a/dev-support/releasedocmaker.py
+++ /dev/null
@@ -1,580 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from glob import glob
-from optparse import OptionParser
-from time import gmtime, strftime
-import pprint
-import os
-import re
-import sys
-import urllib
-import urllib2
-try:
- import json
-except ImportError:
- import simplejson as json
-
-releaseVersion={}
-namePattern = re.compile(r' \([0-9]+\)')
-
-asflicense='''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-def clean(str):
- return tableclean(re.sub(namePattern, "", str))
-
-def formatComponents(str):
- str = re.sub(namePattern, '', str).replace("'", "")
- if str != "":
- ret = str
- else:
- # some markdown parsers don't like empty tables
- ret = "."
- return clean(ret)
-
-# convert to utf-8
-# protect some known md metachars
-# or chars that screw up doxia
-def tableclean(str):
- str=str.encode('utf-8')
- str=str.replace("_","\_")
- str=str.replace("\r","")
- str=str.rstrip()
- return str
-
-# same thing as tableclean,
-# except table metachars are also
-# escaped as well as more
-# things we don't want doxia to
-# screw up
-def notableclean(str):
- str=tableclean(str)
- str=str.replace("|","\|")
- str=str.replace("<","\<")
- str=str.replace(">","\>")
- str=str.replace("*","\*")
- str=str.rstrip()
- return str
-
-# clean output dir
-def cleanOutputDir(dir):
- files = os.listdir(dir)
- for name in files:
- os.remove(os.path.join(dir,name))
- os.rmdir(dir)
-
-def mstr(obj):
- if (obj is None):
- return ""
- return unicode(obj)
-
-def buildindex(title,license):
- versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
- with open("index.md","w") as indexfile:
- if license is True:
- indexfile.write(asflicense)
- for v in versions:
- indexfile.write("* %s v%s\n" % (title,v))
- for k in ("Changes","Release Notes"):
- indexfile.write(" * %s (%s/%s.%s.html)\n" \
- % (k,v,k.upper().replace(" ",""),v))
- indexfile.close()
-
-class GetVersions:
- """ yo """
- def __init__(self,versions, projects):
- versions = versions
- projects = projects
- self.newversions = []
- pp = pprint.PrettyPrinter(indent=4)
- at=0
- end=1
- count=100
- versions.sort()
- print "Looking for %s through %s"%(versions[0],versions[-1])
- for p in projects:
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
- data = json.loads(resp.read())
- for d in data:
- if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]:
- print "Adding %s to the list" % d['name']
- self.newversions.append(d['name'])
- newlist=list(set(self.newversions))
- self.newversions=newlist
-
- def getlist(self):
- pp = pprint.PrettyPrinter(indent=4)
- return(self.newversions)
-
-class Version:
- """Represents a version number"""
- def __init__(self, data):
- self.mod = False
- self.data = data
- found = re.match('^((\d+)(\.\d+)*).*$', data)
- if (found):
- self.parts = [ int(p) for p in found.group(1).split('.') ]
- else:
- self.parts = []
- # backfill version with zeroes if missing parts
- self.parts.extend((0,) * (3 - len(self.parts)))
-
- def __str__(self):
- if (self.mod):
- return '.'.join([ str(p) for p in self.parts ])
- return self.data
-
- def __cmp__(self, other):
- return cmp(self.parts, other.parts)
-
-class Jira:
- """A single JIRA"""
-
- def __init__(self, data, parent):
- self.key = data['key']
- self.fields = data['fields']
- self.parent = parent
- self.notes = None
- self.incompat = None
- self.reviewed = None
-
- def getId(self):
- return mstr(self.key)
-
- def getDescription(self):
- return mstr(self.fields['description'])
-
- def getReleaseNote(self):
- if (self.notes is None):
- field = self.parent.fieldIdMap['Release Note']
- if (self.fields.has_key(field)):
- self.notes=mstr(self.fields[field])
- else:
- self.notes=self.getDescription()
- return self.notes
-
- def getPriority(self):
- ret = ""
- pri = self.fields['priority']
- if(pri is not None):
- ret = pri['name']
- return mstr(ret)
-
- def getAssignee(self):
- ret = ""
- mid = self.fields['assignee']
- if(mid is not None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getComponents(self):
- if (len(self.fields['components'])>0):
- return ", ".join([ comp['name'] for comp in self.fields['components'] ])
- else:
- return ""
-
- def getSummary(self):
- return self.fields['summary']
-
- def getType(self):
- ret = ""
- mid = self.fields['issuetype']
- if(mid is not None):
- ret = mid['name']
- return mstr(ret)
-
- def getReporter(self):
- ret = ""
- mid = self.fields['reporter']
- if(mid is not None):
- ret = mid['displayName']
- return mstr(ret)
-
- def getProject(self):
- ret = ""
- mid = self.fields['project']
- if(mid is not None):
- ret = mid['key']
- return mstr(ret)
-
- def __cmp__(self,other):
- selfsplit=self.getId().split('-')
- othersplit=other.getId().split('-')
- v1=cmp(selfsplit[0],othersplit[0])
- if (v1!=0):
- return v1
- else:
- if selfsplit[1] < othersplit[1]:
- return True
- elif selfsplit[1] > othersplit[1]:
- return False
- return False
-
- def getIncompatibleChange(self):
- if (self.incompat is None):
- field = self.parent.fieldIdMap['Hadoop Flags']
- self.reviewed=False
- self.incompat=False
- if (self.fields.has_key(field)):
- if self.fields[field]:
- for hf in self.fields[field]:
- if hf['value'] == "Incompatible change":
- self.incompat=True
- if hf['value'] == "Reviewed":
- self.reviewed=True
- return self.incompat
-
- def checkMissingComponent(self):
- if (len(self.fields['components'])>0):
- return False
- return True
-
- def checkMissingAssignee(self):
- if (self.fields['assignee'] is not None):
- return False
- return True
-
- def checkVersionString(self):
- field = self.parent.fieldIdMap['Fix Version/s']
- for h in self.fields[field]:
- found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
- if not found:
- return True
- return False
-
- def getReleaseDate(self,version):
- for j in range(len(self.fields['fixVersions'])):
- if self.fields['fixVersions'][j]==version:
- return(self.fields['fixVersions'][j]['releaseDate'])
- return None
-
-class JiraIter:
- """An Iterator of JIRAs"""
-
- def __init__(self, version, projects):
- self.version = version
- self.projects = projects
- v=str(version).replace("-SNAPSHOT","")
-
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field")
- data = json.loads(resp.read())
-
- self.fieldIdMap = {}
- for part in data:
- self.fieldIdMap[part['name']] = part['id']
-
- self.jiras = []
- at=0
- end=1
- count=100
- while (at < end):
- params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count})
- resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
- data = json.loads(resp.read())
- if (data.has_key('errorMessages')):
- raise Exception(data['errorMessages'])
- at = data['startAt'] + data['maxResults']
- end = data['total']
- self.jiras.extend(data['issues'])
-
- needaversion=False
- if v not in releaseVersion:
- needaversion=True
-
- if needaversion is True:
- for i in range(len(data['issues'])):
- for j in range(len(data['issues'][i]['fields']['fixVersions'])):
- if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]:
- releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\
- data['issues'][i]['fields']['fixVersions'][j]['releaseDate']
-
- self.iter = self.jiras.__iter__()
-
- def __iter__(self):
- return self
-
- def next(self):
- data = self.iter.next()
- j = Jira(data, self)
- return j
-
-class Outputs:
- """Several different files to output to at the same time"""
-
- def __init__(self, base_file_name, file_name_pattern, keys, params={}):
- self.params = params
- self.base = open(base_file_name%params, 'w')
- self.others = {}
- for key in keys:
- both = dict(params)
- both['key'] = key
- self.others[key] = open(file_name_pattern%both, 'w')
-
- def writeAll(self, pattern):
- both = dict(self.params)
- both['key'] = ''
- self.base.write(pattern%both)
- for key in self.others.keys():
- both = dict(self.params)
- both['key'] = key
- self.others[key].write(pattern%both)
-
- def writeKeyRaw(self, key, str):
- self.base.write(str)
- if (self.others.has_key(key)):
- self.others[key].write(str)
-
- def close(self):
- self.base.close()
- for fd in self.others.values():
- fd.close()
-
- def writeList(self, mylist):
- for jira in sorted(mylist):
- line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s | %s | %s | %s | %s |\n' \
- % (notableclean(jira.getId()), notableclean(jira.getId()),
- notableclean(jira.getSummary()),
- notableclean(jira.getPriority()),
- formatComponents(jira.getComponents()),
- notableclean(jira.getReporter()),
- notableclean(jira.getAssignee()))
- self.writeKeyRaw(jira.getProject(), line)
-
-def main():
- parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]",
- epilog=
- "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory"
- " named after the highest version provided.")
- parser.add_option("-i","--index", dest="index", action="store_true",
- default=False, help="build an index file")
- parser.add_option("-l","--license", dest="license", action="store_false",
- default=True, help="Add an ASF license")
- parser.add_option("-n","--lint", dest="lint", action="store_true",
- help="use lint flag to exit on failures")
- parser.add_option("-p", "--project", dest="projects",
- action="append", type="string",
- help="projects in JIRA to include in releasenotes", metavar="PROJECT")
- parser.add_option("-r", "--range", dest="range", action="store_true",
- default=False, help="Given versions are a range")
- parser.add_option("-t", "--projecttitle", dest="title",
- type="string",
- help="Title to use for the project (default is Apache PROJECT)")
- parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
- default=False, help="use current date for unreleased versions")
- parser.add_option("-v", "--version", dest="versions",
- action="append", type="string",
- help="versions in JIRA to include in releasenotes", metavar="VERSION")
- (options, args) = parser.parse_args()
-
- if (options.versions is None):
- options.versions = []
-
- if (len(args) > 2):
- options.versions.append(args[2])
-
- if (len(options.versions) <= 0):
- parser.error("At least one version needs to be supplied")
-
- proxy = urllib2.ProxyHandler()
- opener = urllib2.build_opener(proxy)
- urllib2.install_opener(opener)
-
- projects = options.projects
-
- if (options.range is True):
- versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ]
- else:
- versions = [ Version(v) for v in options.versions ]
- versions.sort();
-
- if (options.title is None):
- title=projects[0]
- else:
- title=options.title
-
- haderrors=False
-
- for v in versions:
- vstr=str(v)
- jlist = JiraIter(vstr,projects)
-
- if vstr in releaseVersion:
- reldate=releaseVersion[vstr]
- elif options.usetoday:
- reldate=strftime("%Y-%m-%d", gmtime())
- else:
- reldate="Unreleased"
-
- if not os.path.exists(vstr):
- os.mkdir(vstr)
-
- reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
- "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md",
- [], {"ver":v, "date":reldate, "title":title})
- choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md",
- "%(ver)s/CHANGES.%(key)s.%(ver)s.md",
- [], {"ver":v, "date":reldate, "title":title})
-
- if (options.license is True):
- reloutputs.writeAll(asflicense)
- choutputs.writeAll(asflicense)
-
- relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
- 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n'
- chhead = '# %(title)s Changelog\n\n' \
- '## Release %(ver)s - %(date)s\n'\
- '\n'
-
- reloutputs.writeAll(relhead)
- choutputs.writeAll(chhead)
- errorCount=0
- warningCount=0
- lintMessage=""
- incompatlist=[]
- buglist=[]
- improvementlist=[]
- newfeaturelist=[]
- subtasklist=[]
- tasklist=[]
- testlist=[]
- otherlist=[]
-
- for jira in sorted(jlist):
- if jira.getIncompatibleChange():
- incompatlist.append(jira)
- elif jira.getType() == "Bug":
- buglist.append(jira)
- elif jira.getType() == "Improvement":
- improvementlist.append(jira)
- elif jira.getType() == "New Feature":
- newfeaturelist.append(jira)
- elif jira.getType() == "Sub-task":
- subtasklist.append(jira)
- elif jira.getType() == "Task":
- tasklist.append(jira)
- elif jira.getType() == "Test":
- testlist.append(jira)
- else:
- otherlist.append(jira)
-
- line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \
- % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()),
- notableclean(jira.getSummary()))
-
- if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
- warningCount+=1
- reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
- reloutputs.writeKeyRaw(jira.getProject(), line)
- line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
- lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
- reloutputs.writeKeyRaw(jira.getProject(), line)
-
- if jira.checkVersionString():
- warningCount+=1
- lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
-
- if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
- errorCount+=1
- errorMessage=[]
- jira.checkMissingComponent() and errorMessage.append("component")
- jira.checkMissingAssignee() and errorMessage.append("assignee")
- lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
-
- if (len(jira.getReleaseNote())>0):
- reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
- reloutputs.writeKeyRaw(jira.getProject(), line)
- line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
- reloutputs.writeKeyRaw(jira.getProject(), line)
-
- if (options.lint is True):
- print lintMessage
- print "======================================="
- print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
- if (errorCount>0):
- haderrors=True
- cleanOutputDir(vstr)
- continue
-
- reloutputs.writeAll("\n\n")
- reloutputs.close()
-
- choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(incompatlist)
-
- choutputs.writeAll("\n\n### NEW FEATURES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(newfeaturelist)
-
- choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(improvementlist)
-
- choutputs.writeAll("\n\n### BUG FIXES:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(buglist)
-
- choutputs.writeAll("\n\n### TESTS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(testlist)
-
- choutputs.writeAll("\n\n### SUB-TASKS:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(subtasklist)
-
- choutputs.writeAll("\n\n### OTHER:\n\n")
- choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n")
- choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n")
- choutputs.writeList(otherlist)
- choutputs.writeList(tasklist)
-
- choutputs.writeAll("\n\n")
- choutputs.close()
-
- if options.index:
- buildindex(title,options.license)
-
- if haderrors is True:
- sys.exit(1)
-
-if __name__ == "__main__":
- main()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/shelldocs.py
----------------------------------------------------------------------
diff --git a/dev-support/shelldocs.py b/dev-support/shelldocs.py
deleted file mode 100755
index fc7601a..0000000
--- a/dev-support/shelldocs.py
+++ /dev/null
@@ -1,271 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import re
-import sys
-import string
-from optparse import OptionParser
-
-asflicense='''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-def docstrip(key,string):
- string=re.sub("^## @%s " % key ,"",string)
- string=string.lstrip()
- string=string.rstrip()
- return string
-
-def toc(list):
- tocout=[]
- header=()
- for i in list:
- if header != i.getinter():
- header=i.getinter()
- line=" * %s\n" % (i.headerbuild())
- tocout.append(line)
- line=" * [%s](#%s)\n" % (i.getname().replace("_","\_"),i.getname())
- tocout.append(line)
- return tocout
-
-class ShellFunction:
- def __init__(self):
- self.reset()
-
- def __cmp__(self,other):
- if (self.audience == other.audience):
- if (self.stability == other.stability):
- if (self.replaceb == other.replaceb):
- return(cmp(self.name,other.name))
- else:
- if (self.replaceb == "Yes"):
- return -1
- else:
- return 1
- else:
- if (self.stability == "Stable"):
- return -1
- else:
- return 1
- else:
- if (self.audience == "Public"):
- return -1
- else:
- return 1
-
- def reset(self):
- self.name=None
- self.audience=None
- self.stability=None
- self.replaceb=None
- self.returnt=None
- self.desc=None
- self.params=None
-
- def setname(self,text):
- definition=text.split();
- self.name=definition[1]
-
- def getname(self):
- if (self.name is None):
- return "None"
- else:
- return self.name
-
- def setaudience(self,text):
- self.audience=docstrip("audience",text)
- self.audience=self.audience.capitalize()
-
- def getaudience(self):
- if (self.audience is None):
- return "None"
- else:
- return self.audience
-
- def setstability(self,text):
- self.stability=docstrip("stability",text)
- self.stability=self.stability.capitalize()
-
- def getstability(self):
- if (self.stability is None):
- return "None"
- else:
- return self.stability
-
- def setreplace(self,text):
- self.replaceb=docstrip("replaceable",text)
- self.replaceb=self.replaceb.capitalize()
-
- def getreplace(self):
- if (self.replaceb is None):
- return "None"
- else:
- return self.replaceb
-
- def getinter(self):
- return( (self.getaudience(), self.getstability(), self.getreplace()))
-
- def addreturn(self,text):
- if (self.returnt is None):
- self.returnt = []
- self.returnt.append(docstrip("return",text))
-
- def getreturn(self):
- if (self.returnt is None):
- return "Nothing"
- else:
- return "\n\n".join(self.returnt)
-
- def adddesc(self,text):
- if (self.desc is None):
- self.desc = []
- self.desc.append(docstrip("description",text))
-
- def getdesc(self):
- if (self.desc is None):
- return "None"
- else:
- return " ".join(self.desc)
-
- def addparam(self,text):
- if (self.params is None):
- self.params = []
- self.params.append(docstrip("param",text))
-
- def getparams(self):
- if (self.params is None):
- return ""
- else:
- return " ".join(self.params)
-
- def getusage(self):
- line="%s %s" % (self.name, self.getparams())
- return line
-
- def headerbuild(self):
- if self.getreplace() == "Yes":
- replacetext="Replaceable"
- else:
- replacetext="Not Replaceable"
- line="%s/%s/%s" % (self.getaudience(), self.getstability(), replacetext)
- return(line)
-
- def getdocpage(self):
- line="### `%s`\n\n"\
- "* Synopsis\n\n"\
- "```\n%s\n"\
- "```\n\n" \
- "* Description\n\n" \
- "%s\n\n" \
- "* Returns\n\n" \
- "%s\n\n" \
- "| Classification | Level |\n" \
- "| :--- | :--- |\n" \
- "| Audience | %s |\n" \
- "| Stability | %s |\n" \
- "| Replaceable | %s |\n\n" \
- % (self.getname(),
- self.getusage(),
- self.getdesc(),
- self.getreturn(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
-
- def __str__(self):
- line="{%s %s %s %s}" \
- % (self.getname(),
- self.getaudience(),
- self.getstability(),
- self.getreplace())
- return line
-
-def main():
- parser=OptionParser(usage="usage: %prog --skipprnorep --output OUTFILE --input INFILE [--input INFILE ...]")
- parser.add_option("-o","--output", dest="outfile",
- action="store", type="string",
- help="file to create", metavar="OUTFILE")
- parser.add_option("-i","--input", dest="infile",
- action="append", type="string",
- help="file to read", metavar="INFILE")
- parser.add_option("--skipprnorep", dest="skipprnorep",
- action="store_true", help="Skip Private & Not Replaceable")
-
- (options, args)=parser.parse_args()
-
- allfuncs=[]
- for filename in options.infile:
- with open(filename,"r") as shellcode:
- funcdef=ShellFunction()
- for line in shellcode:
- if line.startswith('## @description'):
- funcdef.adddesc(line)
- elif line.startswith('## @audience'):
- funcdef.setaudience(line)
- elif line.startswith('## @stability'):
- funcdef.setstability(line)
- elif line.startswith('## @replaceable'):
- funcdef.setreplace(line)
- elif line.startswith('## @param'):
- funcdef.addparam(line)
- elif line.startswith('## @return'):
- funcdef.addreturn(line)
- elif line.startswith('function'):
- funcdef.setname(line)
- if options.skipprnorep and \
- funcdef.getaudience() == "Private" and \
- funcdef.getreplace() == "No":
- pass
- else:
- allfuncs.append(funcdef)
- funcdef=ShellFunction()
-
- allfuncs=sorted(allfuncs)
-
- outfile=open(options.outfile, "w")
- outfile.write(asflicense)
- for line in toc(allfuncs):
- outfile.write(line)
-
- outfile.write("\n------\n\n")
-
- header=[]
- for funcs in allfuncs:
- if header != funcs.getinter():
- header=funcs.getinter()
- line="## %s\n" % (funcs.headerbuild())
- outfile.write(line)
- outfile.write(funcs.getdocpage())
- outfile.close()
-
-if __name__ == "__main__":
- main()
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/smart-apply-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
deleted file mode 100755
index ebcb660..0000000
--- a/dev-support/smart-apply-patch.sh
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env bash
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Determine if the git diff patch file has prefixes.
-# These files are generated via "git diff" *without* the --no-prefix option.
-#
-# We can apply these patches more easily because we know that the a/ and b/
-# prefixes in the "diff" lines stands for the project root directory.
-# So we don't have to hunt for the project root.
-# And of course, we know that the patch file was generated using git, so we
-# know git apply can handle it properly.
-#
-# Arguments: git diff file name.
-# Return: 0 if it is a git diff with prefix; 1 otherwise.
-#
-has_prefix() {
- awk '/^diff --git / { if ($3 !~ "^a/" || $4 !~ "^b/") { exit 1 } }
- /^\+{3}|-{3} / { if ($2 !~ "^[ab]/" && $2 !~ "^/dev/null") { exit 1 } }' "$1"
- return $?
-}
-
-PATCH_FILE=$1
-DRY_RUN=$2
-if [ -z "$PATCH_FILE" ]; then
- echo usage: $0 patch-file
- exit 1
-fi
-
-TMPDIR=${TMPDIR:-/tmp}
-PATCH=${PATCH:-patch} # allow overriding patch binary
-
-# Cleanup handler for temporary files
-TOCLEAN=""
-cleanup() {
- rm $TOCLEAN
- exit $1
-}
-trap "cleanup 1" HUP INT QUIT TERM
-
-# Allow passing "-" for stdin patches
-if [ "$PATCH_FILE" == "-" ]; then
- PATCH_FILE="$TMPDIR/smart-apply.in.$RANDOM"
- cat /dev/fd/0 > $PATCH_FILE
- TOCLEAN="$TOCLEAN $PATCH_FILE"
-fi
-
-ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
-if [[ ${PATCH_FILE} =~ ^http || ${PATCH_FILE} =~ ${ISSUE_RE} ]]; then
- # Allow downloading of patches
- PFILE="$TMPDIR/smart-apply.in.$RANDOM"
- TOCLEAN="$TOCLEAN $PFILE"
- if [[ ${PATCH_FILE} =~ ^http ]]; then
- patchURL="${PATCH_FILE}"
- else # Get URL of patch from JIRA
- wget -q -O "${PFILE}" "http://issues.apache.org/jira/browse/${PATCH_FILE}"
- if [[ $? != 0 ]]; then
- echo "Unable to determine what ${PATCH_FILE} may reference." 1>&2
- cleanup 1
- elif [[ $(grep -c 'Patch Available' "${PFILE}") == 0 ]]; then
- echo "${PATCH_FILE} is not \"Patch Available\". Exiting." 1>&2
- cleanup 1
- fi
- relativePatchURL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' "${PFILE}" | grep -v -e 'htm[l]*$' | sort | tail -1 | grep -o '/jira/secure/attachment/[0-9]*/[^"]*')
- patchURL="http://issues.apache.org${relativePatchURL}"
- fi
- if [[ -n $DRY_RUN ]]; then
- echo "Downloading ${patchURL}"
- fi
- wget -q -O "${PFILE}" "${patchURL}"
- if [[ $? != 0 ]]; then
- echo "${PATCH_FILE} could not be downloaded." 1>&2
- cleanup 1
- fi
- PATCH_FILE="${PFILE}"
-fi
-
-# Case for git-diff patches
-if grep -q "^diff --git" "${PATCH_FILE}"; then
- GIT_FLAGS="--binary -v"
- if has_prefix "$PATCH_FILE"; then
- GIT_FLAGS="$GIT_FLAGS -p1"
- else
- GIT_FLAGS="$GIT_FLAGS -p0"
- fi
- if [[ -z $DRY_RUN ]]; then
- GIT_FLAGS="$GIT_FLAGS --stat --apply"
- echo Going to apply git patch with: git apply "${GIT_FLAGS}"
- else
- GIT_FLAGS="$GIT_FLAGS --check"
- fi
- # shellcheck disable=SC2086
- git apply ${GIT_FLAGS} "${PATCH_FILE}"
- if [[ $? == 0 ]]; then
- cleanup 0
- fi
- echo "git apply failed. Going to apply the patch with: ${PATCH}"
-fi
-
-# Come up with a list of changed files into $TMP
-TMP="$TMPDIR/smart-apply.paths.$RANDOM"
-TOCLEAN="$TOCLEAN $TMP"
-
-if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
- PLEVEL=0
- #if the patch applied at P0 there is the possability that all we are doing
- # is adding new files and they would apply anywhere. So try to guess the
- # correct place to put those files.
-
- TMP2="$TMPDIR/smart-apply.paths.2.$RANDOM"
- TOCLEAN="$TOCLEAN $TMP2"
-
- egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort -u > $TMP2
-
- if [ ! -s $TMP2 ]; then
- echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting."
- cleanup 1
- fi
-
- #first off check that all of the files do not exist
- FOUND_ANY=0
- for CHECK_FILE in $(cat $TMP2)
- do
- if [[ -f $CHECK_FILE ]]; then
- FOUND_ANY=1
- fi
- done
-
- if [[ "$FOUND_ANY" = "0" ]]; then
- #all of the files are new files so we have to guess where the correct place to put it is.
-
- # if all of the lines start with a/ or b/, then this is a git patch that
- # was generated without --no-prefix
- if ! grep -qv '^a/\|^b/' $TMP2 ; then
- echo Looks like this is a git patch. Stripping a/ and b/ prefixes
- echo and incrementing PLEVEL
- PLEVEL=$[$PLEVEL + 1]
- sed -i -e 's,^[ab]/,,' $TMP2
- fi
-
- PREFIX_DIRS_AND_FILES=$(cut -d '/' -f 1 $TMP2 | sort -u)
-
- # if we are at the project root then nothing more to do
- if [[ -d hadoop-common-project ]]; then
- echo Looks like this is being run at project root
-
- # if all of the lines start with hadoop-common/, hadoop-hdfs/, hadoop-yarn/ or hadoop-mapreduce/, this is
- # relative to the hadoop root instead of the subproject root, so we need
- # to chop off another layer
- elif [[ "$PREFIX_DIRS_AND_FILES" =~ ^(hadoop-common-project|hadoop-hdfs-project|hadoop-yarn-project|hadoop-mapreduce-project)$ ]]; then
-
- echo Looks like this is relative to project root. Increasing PLEVEL
- PLEVEL=$[$PLEVEL + 1]
-
- elif ! echo "$PREFIX_DIRS_AND_FILES" | grep -vxq 'hadoop-common-project\|hadoop-hdfs-project\|hadoop-yarn-project\|hadoop-mapreduce-project' ; then
- echo Looks like this is a cross-subproject patch. Try applying from the project root
- cleanup 1
- fi
- fi
-elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=1
-elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
- PLEVEL=2
-else
- echo "The patch does not appear to apply with p0 to p2";
- cleanup 1;
-fi
-
-# If this is a dry run then exit instead of applying the patch
-if [[ -n $DRY_RUN ]]; then
- cleanup 0;
-fi
-
-echo Going to apply patch with: $PATCH -p$PLEVEL
-$PATCH -p$PLEVEL -E < $PATCH_FILE
-
-cleanup $?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.d/checkstyle.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh
deleted file mode 100755
index 6311584..0000000
--- a/dev-support/test-patch.d/checkstyle.sh
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin checkstyle
-
-CHECKSTYLE_TIMER=0
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function checkstyle_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test checkstyle
- fi
-}
-
-function checkstyle_mvnrunner
-{
- local logfile=$1
- local output=$2
- local tmp=${PATCH_DIR}/$$.${RANDOM}
- local j
-
- "${MVN}" clean test checkstyle:checkstyle -DskipTests \
- -Dcheckstyle.consoleOutput=true \
- "-D${PROJECT_NAME}PatchProcess" 2>&1 \
- | tee "${logfile}" \
- | ${GREP} ^/ \
- | ${SED} -e "s,${BASEDIR},.,g" \
- > "${tmp}"
-
- # the checkstyle output files are massive, so
- # let's reduce the work by filtering out files
- # that weren't changed. Some modules are
- # MASSIVE and this can cut the output down to
- # by orders of magnitude!!
- for j in ${CHANGED_FILES}; do
- ${GREP} "${j}" "${tmp}" >> "${output}"
- done
-
- rm "${tmp}" 2>/dev/null
-}
-
-function checkstyle_preapply
-{
- local module_suffix
- local modules=${CHANGED_MODULES}
- local module
-
- verify_needed_test checkstyle
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "checkstyle plugin: prepatch"
-
- start_clock
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
-
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-${PATCH_BRANCH}checkstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
- add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} ${module} checkstyle compilation may be broken."
- fi
- popd >/dev/null
- done
-
- # keep track of how much as elapsed for us already
- CHECKSTYLE_TIMER=$(stop_clock)
- return 0
-}
-
-function checkstyle_calcdiffs
-{
- local orig=$1
- local new=$2
- local diffout=$3
- local tmp=${PATCH_DIR}/cs.$$.${RANDOM}
- local count=0
- local j
-
- # first, pull out just the errors
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
- # compare the errors, generating a string of line
- # numbers. Sorry portability: GNU diff makes this too easy
- ${DIFF} --unchanged-line-format="" \
- --old-line-format="" \
- --new-line-format="%dn " \
- "${tmp}.branch" \
- "${tmp}.patch" > "${tmp}.lined"
-
- # now, pull out those lines of the raw output
- # shellcheck disable=SC2013
- for j in $(cat "${tmp}.lined"); do
- # shellcheck disable=SC2086
- head -${j} "${new}" | tail -1 >> "${diffout}"
- done
-
- if [[ -f "${diffout}" ]]; then
- # shellcheck disable=SC2016
- count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
- fi
- rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
- echo "${count}"
-}
-
-function checkstyle_postapply
-{
- local rc=0
- local module
- local modules=${CHANGED_MODULES}
- local module_suffix
- local numprepatch=0
- local numpostpatch=0
- local diffpostpatch=0
-
- verify_needed_test checkstyle
-
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "checkstyle plugin: postpatch"
-
- start_clock
-
- # add our previous elapsed to our new timer
- # by setting the clock back
- offset_clock "${CHECKSTYLE_TIMER}"
-
- for module in ${modules}
- do
- pushd "${module}" >/dev/null
- echo " Running checkstyle in ${module}"
- module_suffix=$(basename "${module}")
-
- checkstyle_mvnrunner \
- "${PATCH_DIR}/maven-patchcheckstyle-${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt"
-
- if [[ $? != 0 ]] ; then
- ((rc = rc +1))
- echo "Post-patch checkstyle compilation is broken."
- add_jira_table -1 checkstyle "Post-patch checkstyle ${module} compilation is broken."
- continue
- fi
-
- #shellcheck disable=SC2016
- diffpostpatch=$(checkstyle_calcdiffs \
- "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" \
- "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" \
- "${PATCH_DIR}/diffcheckstyle${module_suffix}.txt" )
-
- if [[ ${diffpostpatch} -gt 0 ]] ; then
- ((rc = rc + 1))
-
- # shellcheck disable=SC2016
- numprepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}checkstyle${module_suffix}.txt" | ${AWK} '{print $1}')
- # shellcheck disable=SC2016
- numpostpatch=$(wc -l "${PATCH_DIR}/patchcheckstyle${module_suffix}.txt" | ${AWK} '{print $1}')
-
- add_jira_table -1 checkstyle "The applied patch generated "\
- "${diffpostpatch} new checkstyle issues (total was ${numprepatch}, now ${numpostpatch})."
- footer="${footer} @@BASE@@/diffcheckstyle${module_suffix}.txt"
- fi
-
- popd >/dev/null
- done
-
- if [[ ${rc} -gt 0 ]] ; then
- add_jira_footer checkstyle "${footer}"
- return 1
- fi
- add_jira_table +1 checkstyle "There were no new checkstyle issues."
- return 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
deleted file mode 100755
index 9277ea5..0000000
--- a/dev-support/test-patch.d/shellcheck.sh
+++ /dev/null
@@ -1,178 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin shellcheck
-
-SHELLCHECK_TIMER=0
-
-SHELLCHECK=${SHELLCHECK:-$(which shellcheck 2>/dev/null)}
-
-SHELLCHECK_SPECIFICFILES=""
-
-# if it ends in an explicit .sh, then this is shell code.
-# if it doesn't have an extension, we assume it is shell code too
-function shellcheck_filefilter
-{
- local filename=$1
-
- if [[ ${filename} =~ \.sh$ ]]; then
- add_test shellcheck
- SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
- fi
-
- if [[ ! ${filename} =~ \. ]]; then
- add_test shellcheck
- fi
-}
-
-function shellcheck_private_findbash
-{
- local i
- local value
- local list
-
- while read line; do
- value=$(find "${line}" ! -name '*.cmd' -type f \
- | ${GREP} -E -v '(.orig$|.rej$)')
- list="${list} ${value}"
- done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
- # shellcheck disable=SC2086
- echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
-}
-
-function shellcheck_preapply
-{
- local i
-
- verify_needed_test shellcheck
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "shellcheck plugin: prepatch"
-
- if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
- return 0
- fi
-
- start_clock
-
- # shellcheck disable=SC2016
- SHELLCHECK_VERSION=$(${SHELLCHECK} --version | ${GREP} version: | ${AWK} '{print $NF}')
-
- echo "Running shellcheck against all identifiable shell scripts"
- pushd "${BASEDIR}" >/dev/null
- for i in $(shellcheck_private_findbash); do
- if [[ -f ${i} ]]; then
- ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
- fi
- done
- popd > /dev/null
- # keep track of how much as elapsed for us already
- SHELLCHECK_TIMER=$(stop_clock)
- return 0
-}
-
-function shellcheck_calcdiffs
-{
- local orig=$1
- local new=$2
- local diffout=$3
- local tmp=${PATCH_DIR}/sc.$$.${RANDOM}
- local count=0
- local j
-
- # first, pull out just the errors
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${orig}" >> "${tmp}.branch"
-
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${new}" >> "${tmp}.patch"
-
- # compare the errors, generating a string of line
- # numbers. Sorry portability: GNU diff makes this too easy
- ${DIFF} --unchanged-line-format="" \
- --old-line-format="" \
- --new-line-format="%dn " \
- "${tmp}.branch" \
- "${tmp}.patch" > "${tmp}.lined"
-
- # now, pull out those lines of the raw output
- # shellcheck disable=SC2013
- for j in $(cat "${tmp}.lined"); do
- # shellcheck disable=SC2086
- head -${j} "${new}" | tail -1 >> "${diffout}"
- done
-
- if [[ -f "${diffout}" ]]; then
- # shellcheck disable=SC2016
- count=$(wc -l "${diffout}" | ${AWK} '{print $1}' )
- fi
- rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
- echo "${count}"
-}
-
-function shellcheck_postapply
-{
- local i
-
- verify_needed_test shellcheck
- if [[ $? == 0 ]]; then
- return 0
- fi
-
- big_console_header "shellcheck plugin: postpatch"
-
- if [[ ! -x "${SHELLCHECK}" ]]; then
- hadoop_error "shellcheck is not available."
- add_jira_table 0 shellcheck "Shellcheck was not available."
- return 0
- fi
-
- start_clock
-
- # add our previous elapsed to our new timer
- # by setting the clock back
- offset_clock "${SHELLCHECK_TIMER}"
-
- echo "Running shellcheck against all identifiable shell scripts"
- # we re-check this in case one has been added
- for i in $(shellcheck_private_findbash); do
- ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
- done
-
- # shellcheck disable=SC2016
- numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
- # shellcheck disable=SC2016
- numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
-
- diffPostpatch=$(shellcheck_calcdiffs \
- "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
- "${PATCH_DIR}/patchshellcheck-result.txt" \
- "${PATCH_DIR}/diffpatchshellcheck.txt"
- )
-
- if [[ ${diffPostpatch} -gt 0 ]] ; then
- add_jira_table -1 shellcheck "The applied patch generated "\
- "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
- add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
- return 1
- fi
-
- add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
- return 0
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cecad2d/dev-support/test-patch.d/whitespace.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh
deleted file mode 100755
index 324481c..0000000
--- a/dev-support/test-patch.d/whitespace.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-add_plugin whitespace
-
-function whitespace_postapply
-{
- local count
- local j
-
- big_console_header "Checking for whitespace at the end of lines"
- start_clock
-
- pushd "${BASEDIR}" >/dev/null
- for j in ${CHANGED_FILES}; do
- ${GREP} -nHE '[[:blank:]]$' "./${j}" | ${GREP} -f "${GITDIFFLINES}" >> "${PATCH_DIR}/whitespace.txt"
- done
-
- # shellcheck disable=SC2016
- count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
-
- if [[ ${count} -gt 0 ]]; then
- add_jira_table -1 whitespace "The patch has ${count}"\
- " line(s) that end in whitespace. Use git apply --whitespace=fix."
- add_jira_footer whitespace "@@BASE@@/whitespace.txt"
- popd >/dev/null
- return 1
- fi
-
- popd >/dev/null
- add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
- return 0
-}