You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ni...@apache.org on 2007/09/13 23:27:34 UTC
svn commit: r575463 - in /lucene/hadoop/nightly: hudsonBuildHadoopPatch.sh
processHadoopPatchEmail.sh
Author: nigel
Date: Thu Sep 13 14:27:34 2007
New Revision: 575463
URL: http://svn.apache.org/viewvc?rev=575463&view=rev
Log:
Add hudson patch build scripts to svn. Contributed by nigel.
Added:
lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh (with props)
lucene/hadoop/nightly/processHadoopPatchEmail.sh (with props)
Added: lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh
URL: http://svn.apache.org/viewvc/lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh?rev=575463&view=auto
==============================================================================
--- lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh (added)
+++ lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh Thu Sep 13 14:27:34 2007
@@ -0,0 +1,408 @@
+#!/bin/bash
+
+#set -x
+
+### Setup global variables. BUILD_ID is set by Hudson
+baseDir=/export/home/hudson/hudson/jobs/Hadoop-Patch
+GREP=/usr/local/bin/grep
+WGET=/usr/sfw/bin/wget
+PATCH=/usr/local/bin/patch
+PS=/usr/ucb/ps
+JIRA=/export/home/hudson/tools/jira_cli/src/cli/jira
+FINDBUGS_HOME=/export/home/hudson/tools/findbugs/latest
+patchNum=""
+revisionNum=""
+JIRA_COMMENT=""
+JIRA_COMMENT_FOOTER="Console output: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/console
+
+This message is automatically generated."
+
+### Retrieve the defect number
+if [ ! -e $baseDir/current/defectNum ] ; then
+ echo "Could not determine the patch to test. Exiting."
+ cleanupAndExit 0
+fi
+defect=`cat $baseDir/current/defectNum`
+if [ -z "$defect" ] ; then
+ echo "Could not determine the patch to test. Exiting."
+ cleanupAndExit 0
+fi
+
+###############################################################################
+setup () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Testing patch for ${defect}."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+
+ ### Download latest patch file (ignoring .htm and .html)
+ $WGET -q -O $baseDir/current/jira http://issues.apache.org/jira/browse/$defect
+ if [[ `$GREP -c 'Patch Available' $baseDir/current/jira` == 0 ]] ; then
+ echo "$defect is not \"Patch Available\". Exiting."
+ cleanupAndExit 0
+ fi
+ relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $baseDir/current/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
+ patchURL="http://issues.apache.org${relativePatchURL}"
+ patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
+ echo "$defect patch is being downloaded at `date` from"
+ echo "$patchURL"
+ $WGET -q -O $baseDir/current/patch $patchURL
+
+ ### Retrieve Subversion revision number
+ cd $baseDir/workspace/trunk
+ revisionNum=`svn log --limit 1 -q . | $GREP -o "^r[0-9]*"`
+ JIRA_COMMENT="Here are the results of testing the latest attachment
+$patchURL
+against trunk revision $revisionNum."
+
+ ### Copy in any supporting files needed by this process
+ cp -r $baseDir/lib/* ./lib
+ #PENDING: cp -f $baseDir/bin/checkstyle* ./src/test
+
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Pre-building trunk to determine current"
+ echo " number of javac and Findbugs warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -Dfindbugs.home=$FINDBUGS_HOME -Djavac.args="-Xlint -Xmaxwarns 1000" -DHadoopPatchProcess= clean tar findbugs &> $baseDir/current/currentJavacWarnings
+ cp $baseDir/workspace/trunk/build/test/findbugs/*.xml $baseDir/current/currentFindbugsWarnings.xml
+}
+
+###############################################################################
+### Check for @author tags in the patch
+checkAuthor () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Checking there are no @author tags in the patch."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ authorTags=`$GREP -c -i '@author' $baseDir/current/patch`
+ echo "There appear to be $authorTags @author tags in the patch."
+ if [[ $authorTags != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ @author -1. The patch appears to contain $authorTags @author tags which the Hadoop community has agreed to not allow in code contributions."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ @author +1. The patch does not contain any @author tags."
+ return 0
+}
+
+###############################################################################
+### Attempt to apply the patch
+applyPatch () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Applying patch."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ $PATCH -E -p0 < $baseDir/current/patch
+ if [[ $? != 0 ]] ; then
+ echo "PATCH APPLICATION FAILED"
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ patch -1. The patch command could not apply the patch."
+ return 1
+ fi
+ return 0
+}
+
+###############################################################################
+### Check there are no javadoc warnings
+checkJavadocWarnings () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched javadoc warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess= clean javadoc
+ javadocWarnings=`$GREP -c '\[javadoc\] [0-9]* warning' $baseDir/builds/$BUILD_ID/log`
+ echo ""
+ echo ""
+ echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."
+ if [[ $javadocWarnings != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ javadoc -1. The javadoc tool appears to have generated $javadocWarningswarning messages."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ javadoc +1. The javadoc tool did not generate any warning messages."
+return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Javac warnings
+checkJavacWarnings () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched javac warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -Djavac.args="-Xlint -Xmaxwarns 1000" -DHadoopPatchProcess= tar &> $baseDir/current/patchJavacWarnings
+
+ ### Compare current and patch javac warning numbers
+ if [[ -f $baseDir/javacWarnings ]] ; then
+ currentJavacWarnings=`$GREP -o '\[javac\] [0-9]* warning' $baseDir/current/currentJavacWarnings | awk '{total += $2} END {print total}'`
+ patchJavacWarnings=`$GREP -o '\[javac\] [0-9]* warning' $baseDir/current/patchJavacWarnings | awk '{total += $2} END {print total}'`
+ echo ""
+ echo ""
+ echo "There appear to be $currentJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch."
+ if [[ $patchJavacWarnings != "" && $currentJavacWarnings != "" ]] ; then
+ if [[ $patchJavacWarnings > $currentJavacWarnings ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ javac -1. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the trunk's current $currentJavacWarnings warnings)."
+ return 1
+ fi
+ fi
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ javac +1. The applied patch does not generate any new compiler warnings."
+ return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Checkstyle warnings
+checkStyle () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched checkstyle warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "THIS IS NOT IMPLEMENTED YET"
+ echo ""
+ echo ""
+ $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess= checkstyle
+ JIRA_COMMENT_FOOTER="Checkstyle results: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/artifact/trunk/build/test/checkstyle-errors.html
+$JIRA_COMMENT_FOOTER"
+ ### TODO: calculate actual patchStyleErrors
+# patchStyleErrors=0
+# if [[ $patchStyleErrors != 0 ]] ; then
+# JIRA_COMMENT="$JIRA_COMMENT
+#
+# checkstyle -1. The patch generated $patchStyleErrors code style errors."
+# return 1
+# fi
+# JIRA_COMMENT="$JIRA_COMMENT
+#
+# checkstyle +1. The patch generated 0 code style errors."
+ return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Findbugs warnings
+checkFindbugsWarnings () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched Findbugs warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -Dfindbugs.home=$FINDBUGS_HOME -DHadoopPatchProcess= findbugs
+JIRA_COMMENT_FOOTER="Findbugs warnings: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/artifact/trunk/build/test/findbugs/newPatchFindbugsWarnings.html
+$JIRA_COMMENT_FOOTER"
+ cp $baseDir/workspace/trunk/build/test/findbugs/*.xml $baseDir/current/patchFindbugsWarnings.xml
+ $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/1999" \
+ $baseDir/current/currentFindbugsWarnings.xml \
+ $baseDir/current/currentFindbugsWarnings.xml
+ $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
+ $baseDir/current/patchFindbugsWarnings.xml \
+ $baseDir/current/patchFindbugsWarnings.xml
+ $FINDBUGS_HOME/bin/computeBugHistory -output $baseDir/current/findbugsMerge.xml \
+ $baseDir/current/currentFindbugsWarnings.xml \
+ $baseDir/current/patchFindbugsWarnings.xml
+ findbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $baseDir/current/findbugsMerge.xml \
+ $baseDir/workspace/trunk/build/test/findbugs/newPatchFindbugsWarnings.xml | /usr/bin/nawk '{print $1}'`
+ $FINDBUGS_HOME/bin/convertXmlToText -html \
+ $baseDir/workspace/trunk/build/test/findbugs/newPatchFindbugsWarnings.xml \
+ $baseDir/workspace/trunk/build/test/findbugs/newPatchFindbugsWarnings.html
+ if [[ $findbugsWarnings != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ findbugs -1. The patch appears to introduce $findbugsWarnings new Findbugs warnings."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ findbugs +1. The patch does not introduce any new Findbugs warnings."
+ return 0
+}
+
+###############################################################################
+### Run the test-core target
+runCoreTests () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Running core tests."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ ### Kill any rogue build processes from the last attempt
+ $PS -auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /usr/bin/kill -9 {} > /dev/null
+
+ $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=yes tar test-core
+ if [[ $? != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ core tests -1. The patch failed core unit tests."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ core tests +1. The patch passed core unit tests."
+ return 0
+}
+
+###############################################################################
+### Run the test-contrib target
+runContribTests () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Running contrib tests."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ ### Kill any rogue build processes from the last attempt
+ $PS -auxwww | $GREP HadoopPatchProcess | /usr/bin/nawk '{print $2}' | /usr/bin/xargs -t -I {} /usr/bin/kill -9 {} > /dev/null
+
+ $ANT_HOME/bin/ant -Dversion=${revisionNum}_${defect}_PATCH-${patchNum} -DHadoopPatchProcess= -Dtest.junit.output.format=xml -Dtest.output=yes test-contrib
+ if [[ $? != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ contrib tests -1. The patch failed contrib unit tests."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ contrib tests +1. The patch passed contrib unit tests."
+ return 0
+}
+
+###############################################################################
+### Submit a comment to the defect's Jira
+submitJiraComment () {
+ local result=$1
+ if [[ $result == 0 ]] ; then
+ comment="+1 overall. $JIRA_COMMENT
+
+$JIRA_COMMENT_FOOTER"
+ else
+ comment="-1 overall. $JIRA_COMMENT
+
+$JIRA_COMMENT_FOOTER"
+ fi
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Adding comment to Jira."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "$comment"
+
+ ### Update Jira with a comment
+ export USER=hudson
+# $JIRA -s issues.apache.org/jira login hadoopqa `cat ~/private/hadoopqaJiraPassword`
+# $JIRA -s issues.apache.org/jira comment $defect "$comment"
+}
+
+###############################################################################
+### Cleanup files
+cleanupAndExit () {
+ local result=$1
+ if [ ! -z "$defect" ] ; then
+ if [ -e $baseDir/$defect ] ; then
+ rm -rf $baseDir/$defect
+ fi
+ fi
+ if [ -e $baseDir/current ] ; then
+ rm -f $baseDir/current
+ fi
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Finished build."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ exit $result
+}
+
+###############################################################################
+###############################################################################
+###############################################################################
+
+setup
+checkAuthor
+RESULT=$?
+applyPatch
+if [[ $? != 0 ]] ; then
+ submitJiraComment 1
+ cleanupAndExit $?
+fi
+checkJavadocWarnings
+(( RESULT = RESULT + $? ))
+checkJavacWarnings
+(( RESULT = RESULT + $? ))
+checkStyle
+(( RESULT = RESULT + $? ))
+checkFindbugsWarnings
+(( RESULT = RESULT + $? ))
+runCoreTests
+(( RESULT = RESULT + $? ))
+runContribTests
+(( RESULT = RESULT + $? ))
+JIRA_COMMENT_FOOTER="Test results: http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/$BUILD_NUMBER/testReport/
+$JIRA_COMMENT_FOOTER"
+
+submitJiraComment $RESULT
+cleanupAndExit $RESULT
+
Propchange: lucene/hadoop/nightly/hudsonBuildHadoopPatch.sh
------------------------------------------------------------------------------
svn:executable = *
Added: lucene/hadoop/nightly/processHadoopPatchEmail.sh
URL: http://svn.apache.org/viewvc/lucene/hadoop/nightly/processHadoopPatchEmail.sh?rev=575463&view=auto
==============================================================================
--- lucene/hadoop/nightly/processHadoopPatchEmail.sh (added)
+++ lucene/hadoop/nightly/processHadoopPatchEmail.sh Thu Sep 13 14:27:34 2007
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+#set -x
+
+baseDir=/export/home/hudson/hudson/jobs/Hadoop-Patch
+LOG=$baseDir/log.txt
+
+### Scan email
+while read line
+do
+ ### Look for first occurance of issue number
+ if [[ `echo $line | /usr/local/bin/grep -c "HADOOP-"` == 1 ]] ; then
+ if [[ -z $defect ]] ; then
+ defect=`expr "$line" : '.*\(HADOOP-[0-9]*\)'`
+ fi
+ fi
+ ### Check to see if this issue was just made "Patch Available"
+ if [[ `echo $line | /usr/local/bin/grep -c "Status: Patch Available"` == 1 ]] ; then
+ patch=true
+ break
+ fi
+done
+
+### If this email indicates a new patch, start a build
+if [[ -n $patch && ! -d $baseDir/$defect ]] ; then
+ echo "$defect is being processed at `date`" >> $LOG
+ mkdir $baseDir/$defect
+
+ ### Write the defect number to a file so buildTest.sh
+ ### knows which patch to test.
+ echo $defect > $baseDir/$defect/defectNum
+
+ ### Write the process number of this script to a file so
+ ### it can be killed by a human if necessary.
+ echo $$ > $baseDir/$defect/emailProcessingProcessNumber
+
+ ### Since this script is run by the 'daemon' user by sendmail,
+ ### make sure everything it creates is group writable
+ chmod -R g+w $baseDir/$defect
+
+ ### Wait here until the current build is complete.
+ ### The 'current' link provides a locking mechanism
+ ### between this script and the buildTest.sh script.
+ while [[ -L $baseDir/current ]] ; do
+ sleep 59
+ done
+
+ ### check to make sure the defect dir still exists; this allows
+ ### an admin to delete the directory and bypass the testing
+ if [[ -d $baseDir/$defect ]] ; then
+ ### Create the 'current' link to this build
+ ln -s $baseDir/$defect $baseDir/current
+
+ ### Start build. The 'token' is a secret.
+ echo "$defect patch submitted for testing at `date`" >> $LOG
+ token=`cat ~/private/hudsonBuildToken`
+ /usr/sfw/bin/wget -q -O $baseDir/$defect/build http://lucene.zones.apache.org:8080/hudson/job/Hadoop-Patch/build?token=$token
+ chmod -R g+w $baseDir/$defect
+ fi
+fi
+exit 0
Propchange: lucene/hadoop/nightly/processHadoopPatchEmail.sh
------------------------------------------------------------------------------
svn:executable = *