You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2011/08/09 22:28:49 UTC
svn commit: r1155551 - in /hadoop/common/trunk: dev-support/test-patch.sh
hadoop-common/CHANGES.txt
Author: tomwhite
Date: Tue Aug 9 20:28:48 2011
New Revision: 1155551
URL: http://svn.apache.org/viewvc?rev=1155551&view=rev
Log:
HADOOP-7525. Make arguments to test-patch optional.
Modified:
hadoop/common/trunk/dev-support/test-patch.sh
hadoop/common/trunk/hadoop-common/CHANGES.txt
Modified: hadoop/common/trunk/dev-support/test-patch.sh
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/dev-support/test-patch.sh?rev=1155551&r1=1155550&r2=1155551&view=diff
==============================================================================
--- hadoop/common/trunk/dev-support/test-patch.sh (original)
+++ hadoop/common/trunk/dev-support/test-patch.sh Tue Aug 9 20:28:48 2011
@@ -21,91 +21,151 @@ ulimit -n 1024
bindir=$(dirname $0)
. $bindir/test-patch.properties
+# Defaults
if [ -z "$MAVEN_HOME" ]; then
MVN=mvn
else
MVN=$MAVEN_HOME/bin/mvn
fi
+PROJECT_NAME=Hadoop
+JENKINS=false
+PATCH_DIR=/tmp
+SUPPORT_DIR=/tmp
+BASEDIR=$(pwd)
+
+PS=${PS:-ps}
AWK=${AWK:-awk}
+WGET=${WGET:-wget}
+SVN=${SVN:-svn}
+GREP=${GREP:-grep}
+PATCH=${PATCH:-patch}
+JIRACLI=${JIRA:-jira}
+FINDBUGS_HOME=${FINDBUGS_HOME}
+FORREST_HOME=${FORREST_HOME}
+ECLIPSE_HOME=${ECLIPSE_HOME}
+
+###############################################################################
+printUsage() {
+ echo "Usage: $0 [options] patch-file | defect-number"
+ echo
+ echo "Where:"
+ echo " patch-file is a local patch file containing the changes to test"
+ echo " defect-number is a JIRA defect number (e.g. 'HADOOP-1234') to test (Jenkins only)"
+ echo
+ echo "Options:"
+ echo "--patch-dir=<dir> The directory for working and output files (default '/tmp')"
+ echo "--basedir=<dir> The directory to apply the patch to (default current directory)"
+ echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')"
+ echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')"
+ echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
+ echo "--svn-cmd=<cmd> The 'svn' command to use (default 'svn')"
+ echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
+ echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
+ echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
+ echo "--forrest-home=<path> Forrest home directory (default FORREST_HOME environment variable)"
+ echo "--dirty-workspace Allow the local SVN workspace to have uncommitted changes"
+ echo
+ echo "Jenkins-only options:"
+ echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
+ echo "--support-dir=<dir> The directory to find support files in"
+ echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
+ echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
+ echo "--jira-password=<pw> The password for the 'jira' command"
+ echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
+}
###############################################################################
parseArgs() {
- case "$1" in
- HUDSON)
- ### Set HUDSON to true to indicate that this script is being run by Hudson
- HUDSON=true
- if [[ $# != 16 ]] ; then
- echo "ERROR: usage $0 HUDSON <PATCH_DIR> <SUPPORT_DIR> <PS_CMD> <WGET_CMD> <JIRACLI> <SVN_CMD> <GREP_CMD> <PATCH_CMD> <FINDBUGS_HOME> <FORREST_HOME> <ECLIPSE_HOME> <WORKSPACE_BASEDIR> <JIRA_PASSWD> <CURL_CMD> <DEFECT> "
- cleanupAndExit 0
- fi
- PATCH_DIR=$2
- SUPPORT_DIR=$3
- PS=$4
- WGET=$5
- JIRACLI=$6
- SVN=$7
- GREP=$8
- PATCH=$9
- FINDBUGS_HOME=${10}
- FORREST_HOME=${11}
- ECLIPSE_HOME=${12}
- BASEDIR=${13}
- JIRA_PASSWD=${14}
- CURL=${15}
- defect=${16}
-
- ### Retrieve the defect number
- if [ -z "$defect" ] ; then
- echo "Could not determine the patch to test. Exiting."
- cleanupAndExit 0
- fi
-
- if [ ! -e "$PATCH_DIR" ] ; then
- mkdir -p $PATCH_DIR
- fi
-
- ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME"
+ for i in $*
+ do
+ case $i in
+ --jenkins)
+ JENKINS=true
;;
- DEVELOPER)
- ### Set HUDSON to false to indicate that this script is being run by a developer
- HUDSON=false
- if [[ $# != 9 ]] ; then
- echo "ERROR: usage $0 DEVELOPER <PATCH_FILE> <SCRATCH_DIR> <SVN_CMD> <GREP_CMD> <PATCH_CMD> <FINDBUGS_HOME> <FORREST_HOME> <WORKSPACE_BASEDIR>"
- cleanupAndExit 0
- fi
- ### PATCH_FILE contains the location of the patchfile
- PATCH_FILE=$2
- if [[ ! -e "$PATCH_FILE" ]] ; then
- echo "Unable to locate the patch file $PATCH_FILE"
- cleanupAndExit 0
- fi
- PATCH_DIR=$3
- ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
- if [[ ! -e "$PATCH_DIR" ]] ; then
- mkdir "$PATCH_DIR"
- if [[ $? == 0 ]] ; then
- echo "$PATCH_DIR has been created"
- else
- echo "Unable to create $PATCH_DIR"
- cleanupAndExit 0
- fi
- fi
- SVN=$4
- GREP=$5
- PATCH=$6
- FINDBUGS_HOME=$7
- FORREST_HOME=$8
- BASEDIR=$9
- PS=${PS:-ps}
- ### Obtain the patch filename to append it to the version number
- defect=`basename $PATCH_FILE`
+ --patch-dir=*)
+ PATCH_DIR=${i#*=}
+ ;;
+ --support-dir=*)
+ SUPPORT_DIR=${i#*=}
+ ;;
+ --basedir=*)
+ BASEDIR=${i#*=}
+ ;;
+ --mvn-cmd=*)
+ MVN=${i#*=}
+ ;;
+ --ps-cmd=*)
+ PS=${i#*=}
+ ;;
+ --awk-cmd=*)
+ AWK=${i#*=}
+ ;;
+ --wget-cmd=*)
+ WGET=${i#*=}
+ ;;
+ --svn-cmd=*)
+ SVN=${i#*=}
+ ;;
+ --grep-cmd=*)
+ GREP=${i#*=}
+ ;;
+ --patch-cmd=*)
+ PATCH=${i#*=}
+ ;;
+ --jira-cmd=*)
+ JIRACLI=${i#*=}
+ ;;
+ --jira-password=*)
+ JIRA_PASSWD=${i#*=}
+ ;;
+ --findbugs-home=*)
+ FINDBUGS_HOME=${i#*=}
+ ;;
+ --forrest-home=*)
+ FORREST_HOME=${i#*=}
+ ;;
+ --eclipse-home=*)
+ ECLIPSE_HOME=${i#*=}
+ ;;
+ --dirty-workspace)
+ DIRTY_WORKSPACE=true
;;
*)
- echo "ERROR: usage $0 HUDSON [args] | DEVELOPER [args]"
- cleanupAndExit 0
+ PATCH_OR_DEFECT=$i
;;
- esac
+ esac
+ done
+ if [ -z "$PATCH_OR_DEFECT" ]; then
+ printUsage
+ exit 1
+ fi
+ if [[ $JENKINS == "true" ]] ; then
+ echo "Running in Jenkins mode"
+ defect=$PATCH_OR_DEFECT
+ ECLIPSE_PROPERTY="-Declipse.home=$ECLIPSE_HOME"
+ else
+ echo "Running in developer mode"
+ JENKINS=false
+ ### PATCH_FILE contains the location of the patchfile
+ PATCH_FILE=$PATCH_OR_DEFECT
+ if [[ ! -e "$PATCH_FILE" ]] ; then
+ echo "Unable to locate the patch file $PATCH_FILE"
+ cleanupAndExit 0
+ fi
+ ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
+ if [[ ! -e "$PATCH_DIR" ]] ; then
+ mkdir "$PATCH_DIR"
+ if [[ $? == 0 ]] ; then
+ echo "$PATCH_DIR has been created"
+ else
+ echo "Unable to create $PATCH_DIR"
+ cleanupAndExit 0
+ fi
+ fi
+ ### Obtain the patch filename to append it to the version number
+ defect=`basename $PATCH_FILE`
+ fi
}
###############################################################################
@@ -120,9 +180,10 @@ checkout () {
echo ""
echo ""
### When run by a developer, if the workspace contains modifications, do not continue
+ ### unless the --dirty-workspace option was set
status=`$SVN stat --ignore-externals | sed -e '/^X[ ]*/D'`
- if [[ $HUDSON == "false" ]] ; then
- if [[ "$status" != "" ]] ; then
+ if [[ $JENKINS == "false" ]] ; then
+ if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
echo "ERROR: can't run in a workspace that contains the following modifications"
echo "$status"
cleanupAndExit 1
@@ -140,7 +201,7 @@ checkout () {
###############################################################################
setup () {
### Download latest patch file (ignoring .htm and .html) when run from patch process
- if [[ $HUDSON == "true" ]] ; then
+ if [[ $JENKINS == "true" ]] ; then
$WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect
if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then
echo "$defect is not \"Patch Available\". Exiting."
@@ -188,8 +249,7 @@ setup () {
echo "======================================================================"
echo ""
echo ""
-# echo "$ANT_HOME/bin/ant -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
-# $ANT_HOME/bin/ant -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -D${PROJECT_NAME}PatchProcess= clean tar > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
+ echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1"
$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
if [[ $? != 0 ]] ; then
echo "Trunk compilation is broken?"
@@ -238,7 +298,7 @@ checkTests () {
testReferences=`$GREP -c -i '/test' $PATCH_DIR/patch`
echo "There appear to be $testReferences test files referenced in the patch."
if [[ $testReferences == 0 ]] ; then
- if [[ $HUDSON == "true" ]] ; then
+ if [[ $JENKINS == "true" ]] ; then
patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira`
if [[ $patchIsDoc != 0 ]] ; then
echo "The patch appears to be a documentation patch that doesn't require tests."
@@ -306,10 +366,9 @@ checkJavadocWarnings () {
echo "======================================================================"
echo ""
echo ""
- echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt"
+ echo "$MVN clean compile javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1"
(cd hadoop-project; mvn install)
(cd hadoop-annotations; mvn install)
- #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= clean javadoc | tee $PATCH_DIR/patchJavadocWarnings.txt
$MVN clean compile javadoc:javadoc -DskipTests -Pdocs -D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt | $AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
echo ""
@@ -341,8 +400,7 @@ checkJavacWarnings () {
echo "======================================================================"
echo ""
echo ""
- #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
- #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Djavac.args="-Xlint -Xmaxwarns 1000" $ECLIPSE_PROPERTY -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= clean tar > $PATCH_DIR/patchJavacWarnings.txt 2>&1
+ echo "$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
$MVN clean compile -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch > $PATCH_DIR/patchJavacWarnings.txt 2>&1
if [[ $? != 0 ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
@@ -382,8 +440,7 @@ checkReleaseAuditWarnings () {
echo "======================================================================"
echo ""
echo ""
- #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/patchReleaseAuditWarnings.txt 2>&1"
- #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= releaseaudit > $PATCH_DIR/patchReleaseAuditWarnings.txt 2>&1
+ echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
find . -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt
@@ -427,8 +484,7 @@ checkStyle () {
echo "THIS IS NOT IMPLEMENTED YET"
echo ""
echo ""
- #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= checkstyle"
- #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= checkstyle
+ echo "$MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess"
$MVN compile checkstyle:checkstyle -D${PROJECT_NAME}PatchProcess
JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html
@@ -460,8 +516,7 @@ checkFindbugsWarnings () {
echo "======================================================================"
echo ""
echo ""
- #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=$FINDBUGS_HOME -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs"
- #$ANT_HOME/bin/ant -Dversion="${VERSION}" -Dfindbugs.home=${FINDBUGS_HOME} -Dforrest.home=${FORREST_HOME} -DHadoopPatchProcess= findbugs
+ echo "$MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess"
$MVN clean compile findbugs:findbugs -D${PROJECT_NAME}PatchProcess
if [ $? != 0 ] ; then
@@ -510,15 +565,14 @@ runCoreTests () {
echo ""
### Kill any rogue build processes from the last attempt
- $PS auxwww | $GREP HadoopPatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+ $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
PreTestTarget=""
if [[ $defect == MAPREDUCE-* ]] ; then
PreTestTarget="create-c++-configure"
fi
- #echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME $PreTestTarget test-core"
- #$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME $PreTestTarget test-core
- $MVN clean test -Pnative -DHadoopPatchProcess
+ echo "$MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess"
+ $MVN clean test -Pnative -D${PROJECT_NAME}PatchProcess
if [[ $? != 0 ]] ; then
### Find and format names of failed tests
failed_tests=`grep -l -E "<failure|<error" $WORKSPACE/trunk/target/hadoop-common/surefire-reports/*.xml | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
@@ -553,7 +607,7 @@ runContribTests () {
fi
### Kill any rogue build processes from the last attempt
- $PS auxwww | $GREP HadoopPatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+ $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
#echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib"
#$ANT_HOME/bin/ant -Dversion="${VERSION}" $ECLIPSE_PROPERTY -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no test-contrib
@@ -584,7 +638,7 @@ checkInjectSystemFaults () {
echo ""
### Kill any rogue build processes from the last attempt
- $PS auxwww | $GREP HadoopPatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
+ $PS auxwww | $GREP ${PROJECT_NAME}PatchProcess | $AWK '{print $2}' | /usr/bin/xargs -t -I {} /bin/kill -9 {} > /dev/null
#echo "$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults"
#$ANT_HOME/bin/ant -Dversion="${VERSION}" -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=no -Dcompile.c++=yes -Dforrest.home=$FORREST_HOME inject-system-faults
@@ -606,7 +660,7 @@ checkInjectSystemFaults () {
submitJiraComment () {
local result=$1
### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer
- if [[ $HUDSON == "false" ]] ; then
+ if [[ $JENKINS == "false" ]] ; then
JIRA_COMMENT_FOOTER=""
fi
if [[ $result == 0 ]] ; then
@@ -625,7 +679,7 @@ $JIRA_COMMENT_FOOTER"
$comment"
- if [[ $HUDSON == "true" ]] ; then
+ if [[ $JENKINS == "true" ]] ; then
echo ""
echo ""
echo "======================================================================"
@@ -646,7 +700,7 @@ $comment"
### Cleanup files
cleanupAndExit () {
local result=$1
- if [[ $HUDSON == "true" ]] ; then
+ if [[ $JENKINS == "true" ]] ; then
if [ -e "$PATCH_DIR" ] ; then
mv $PATCH_DIR $BASEDIR
fi
@@ -678,7 +732,7 @@ cd $BASEDIR
checkout
RESULT=$?
-if [[ $HUDSON == "true" ]] ; then
+if [[ $JENKINS == "true" ]] ; then
if [[ $RESULT != 0 ]] ; then
exit 100
fi
@@ -687,7 +741,7 @@ setup
checkAuthor
RESULT=$?
-if [[ $HUDSON == "true" ]] ; then
+if [[ $JENKINS == "true" ]] ; then
cleanUpXml
fi
checkTests
@@ -709,7 +763,7 @@ checkFindbugsWarnings
checkReleaseAuditWarnings
(( RESULT = RESULT + $? ))
### Do not call these when run by a developer
-if [[ $HUDSON == "true" ]] ; then
+if [[ $JENKINS == "true" ]] ; then
runCoreTests
(( RESULT = RESULT + $? ))
runContribTests
Modified: hadoop/common/trunk/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common/CHANGES.txt?rev=1155551&r1=1155550&r2=1155551&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common/CHANGES.txt Tue Aug 9 20:28:48 2011
@@ -309,6 +309,8 @@ Trunk (unreleased changes)
HADOOP-7501. Publish Hadoop Common artifacts (post HADOOP-6671) to Apache
SNAPSHOTs repo. (Alejandro Abdelnur via tomwhite)
+ HADOOP-7525. Make arguments to test-patch optional. (tomwhite)
+
OPTIMIZATIONS
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole