You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ni...@apache.org on 2010/10/22 08:49:58 UTC
svn commit: r1026225 - /hadoop/nightly/hudsonPatchQueueAdmin.sh
Author: nigel
Date: Fri Oct 22 06:49:58 2010
New Revision: 1026225
URL: http://svn.apache.org/viewvc?rev=1026225&view=rev
Log:
HADOOP-7003: New script for managing queue of pre-commit patches that need testing. Contributed by nigel.
Modified:
hadoop/nightly/hudsonPatchQueueAdmin.sh
Modified: hadoop/nightly/hudsonPatchQueueAdmin.sh
URL: http://svn.apache.org/viewvc/hadoop/nightly/hudsonPatchQueueAdmin.sh?rev=1026225&r1=1026224&r2=1026225&view=diff
==============================================================================
--- hadoop/nightly/hudsonPatchQueueAdmin.sh (original)
+++ hadoop/nightly/hudsonPatchQueueAdmin.sh Fri Oct 22 06:49:58 2010
@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/bin/bash -e
-usage()
+usage()
{
cat << EOF
usage: $0 options
@@ -9,55 +9,38 @@ This script initates patch build's on Hu
OPTIONS:
-h Show this message
- -p Project Name like HADOOP, PIG or ZOOPKEEPER.
- -s Build Server Names
- -c Called by values like USER OR Build Server name
- -t Path to curl binary
- -S Path to sed binary
- -u Build url token
- -q Path to the Queue Directory
- ### The directory to accumulate the patch queue. Must be
- ### writable by this process.
+ -f Url to Jira XML filter of patch available issues.
+ -c Path to curl binary
+ -s Path to sed binary
+ -t Build token
-v Verbose
EOF
}
-PROJECT=
-QUEUE_DIR=
-BUILD_SERVERS=
-CALLED_BY=
+FILTER=
CURL=
-BUILD_URL_TOKEN=
+BUILD_TOKEN=
VERBOSE=
SED=
-while getopts "hp:s:c:t:u:q:S:v" OPTION
+while getopts "hf:t:s:u:v" OPTION
do
case $OPTION in
h)
usage
exit 1
;;
- p)
- PROJECT=$OPTARG
- ;;
- s)
- BUILD_SERVERS=$OPTARG
+ f)
+ FILTER=$OPTARG
;;
c)
- CALLED_BY=$OPTARG
- ;;
- t)
CURL=$OPTARG
;;
- S)
+ s)
SED=$OPTARG
;;
- u)
- BUILD_URL_TOKEN=$OPTARG
- ;;
- q)
- QUEUE_DIR=$OPTARG
+ t)
+ BUILD_TOKEN=$OPTARG
;;
v)
VERBOSE=1
@@ -69,72 +52,44 @@ do
esac
done
-if [[ -z $PROJECT ]] || [[ -z $BUILD_SERVERS ]] || [[ -z $CALLED_BY ]] || [[ -z $SED ]] || [[ -z $CURL ]] || [[ -z $BUILD_URL_TOKEN ]] || [[ -z $QUEUE_DIR ]]
-then
- usage
- exit 1
+if [[ -z $FILTER ]] || [[ -z $SED ]] || [[ -z $CURL ]] || [[ -z $BUILD_TOKEN ]] ; then
+ usage
+ exit 1
fi
-if [[ $VERBOSE == 1 ]]; then
-set -x
+if [[ $VERBOSE == 1 ]] ; then
+ set -x
fi
-PATCH_QUEUE=${QUEUE_DIR}/patch_queue.txt
-
-### To check if this patch_admin job called by test-patch.sh or hudson
-### if called by test-patch.sh then freee up the slae machine.
-if [[ $CALLED_BY != "" && $CALLED_BY != "USER" ]]; then
- rm -rf ${QUEUE_DIR}/${CALLED_BY}
-fi
+### Grab the latest patch available query from Jira
+curl --fail --location --retry 3 --output patch_available.xml $FILTER
-QUEUE_HTML_FILE=`pwd`/${PROJECT}_PatchQueue.html
+### Grab all the key (issue numbers) and largest attachment id for each item in the XML
+xpath -q -e "//item/key/text() | //item/attachments/attachment[not(../attachment/@id > @id)]/@id" patch_available.xml |tee patch_available2.elements
-echo "<html>" > $QUEUE_HTML_FILE
-echo "<title>Patch Queue for $PROJECT</title>" >> $QUEUE_HTML_FILE
-echo "<h3 align='left'><img src="http://hadoop.apache.org/core/images/hadoop-logo.jpg" height="50"></img>Patch Queue for ${PROJECT}</h3>" >> $QUEUE_HTML_FILE
-echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >> $QUEUE_HTML_FILE
-echo "<h4>Currently Running (or Waiting To Run)</h4>" >> $QUEUE_HTML_FILE
-echo "<table cellspacing=10><tr align=left><th>Issue</th><th>Submitted to</th><th>Date Submitted to Run</th></tr>" >> $QUEUE_HTML_FILE
-
-for SLAVE in $BUILD_SERVERS
-do
- TRIGGER_BUILD_URL=${HUDSON_URL}'job/'${PROJECT}'-Patch-'${SLAVE}${BUILD_URL_TOKEN}
- CURRENT_PATCH=${QUEUE_DIR}/${SLAVE}
- defect=`head -n 1 $PATCH_QUEUE | awk '{print $1}'`
-if [[ ! -f $CURRENT_PATCH ]]; then
- if [[ $defect ]]; then
- echo $defect > ${CURRENT_PATCH}
- $SED -i '1d' $PATCH_QUEUE
- echo "$defect patch submitted for testing at `date +"%b %a %H:%M"`"
- $CURL "$TRIGGER_BUILD_URL&DEFECTNUM=$defect"
- echo "<tr><td><a href=\"http://issues.apache.org/jira/browse/${defect}\">$defect</a></td><td>${SLAVE}</td><td>`date +"%b %a %H:%M"` EST</td></tr>" >> $QUEUE_HTML_FILE
- else
- CURRENT_PATCH="none"
- CURRENT_PATCH_TIME="-"
- echo "<tr><td>$CURRENT_PATCH</td><td>$CURRENT_PATCH_TIME</td></tr>" >> $QUEUE_HTML_FILE
+### Replace newlines with nothing, then replace id=" with =, then replace " with newline
+### to yield lines with pairs (issueNumber,largestAttachmentId). Example: HADOOP-123,456984
+awk '{ printf "%s", $0 }' patch_available2.elements | sed -e "s/\W*id=\"/,/g" | sed -e "s/\"/\n/g" |tee patch_available3.txt
+
+### Grab the latest patch_tested.txt file from this job on Hudson
+curl --fail --location --retry 3 --output patch_tested.txt ${JOB_URL}/lastSuccessfulBuild/artifact/patch_tested.txt
+
+### Iterate through issue list and find the (issueNumber,largestAttachmentId) pairs that have
+### not been tested (ie don't already exist in the patch_tested.txt file
+touch patch_tested.txt
+cat patch_available3.txt | while read PAIR ; do
+ set +e
+ COUNT=`grep -c "$PAIR" patch_tested.txt`
+ set -e
+ if [ "$COUNT" -lt "1" ] ; then
+ ### Parse $PAIR into project, issue number, and attachment id
+ PROJECT=`echo $PAIR | sed -e "s/-.*//g"`
+ ISSUE=`echo $PAIR | sed -e "s/.*-//g" | sed -e "s/,.*//g"`
+ ATTACHMENT=`echo $PAIR | sed -e "s/.*,//g"`
+ ### Kick off job
+ echo "Starting job $PROJECT with issue $ISSUE and attachment $ATTACHMENT"
+ curl --fail --location --retry 3 "${HUDSON_URL}/job/PreCommit-${PROJECT}-Build/buildWithParameters?token=${BUILD_TOKEN}&ISSUE_NUM=${ISSUE}&ATTACHMENT_ID=${ATTACHMENT}"
+ ### Mark this pair as tested by appending to file
+ echo "$PAIR" >> patch_tested.txt
fi
-else
- CURRENT_PATCH_TIME=`/bin/ls -dtl ${CURRENT_PATCH} | awk '{print $6" "$7" "$8" EST"}'`
- EXEC_PATCH=`cat ${CURRENT_PATCH}`
- echo "<tr><td><a href=\"http://issues.apache.org/jira/browse/$EXEC_PATCH\">$EXEC_PATCH</a></td><td>${SLAVE}</td><td>$CURRENT_PATCH_TIME</td></tr>" >> $QUEUE_HTML_FILE
-fi
done
-echo "</table>" >> $QUEUE_HTML_FILE
-echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >> $QUEUE_HTML_FILE
-echo "<h4>Waiting in the Queue</h4>" >> $QUEUE_HTML_FILE
-echo "<table cellspacing=10><tr align=left><th>Issue</th><th>Date Submitted</th></tr>" >> $QUEUE_HTML_FILE
-
-if [ -s $PATCH_QUEUE ]; then
- while read PATCH_DTLS
- do
- TEMP_QUEUE_DTLS=`echo $PATCH_DTLS | awk '{print "<tr><td><a href=\"http://issues.apache.org/jira/browse/"$1"\">"$1"</a></td><td>"$2" "$3" "$4" "$5" EST</td></tr>"}'`
- echo $TEMP_QUEUE_DTLS >> $QUEUE_HTML_FILE
- done < "$PATCH_QUEUE"
-else
- echo "<tr><td>empty</td><td>-</td></tr>" >> $QUEUE_HTML_FILE
-fi
-
-echo "</table>" >> $QUEUE_HTML_FILE
-echo "<hr style='height:2px;border-width:0;color:red;background-color:blue'>" >> $QUEUE_HTML_FILE
-echo "This file last updated at: `date`" >>$QUEUE_HTML_FILE
-echo "</html>" >> $QUEUE_HTML_FILE