You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2012/08/20 19:51:37 UTC
svn commit: r1375141 - in /incubator/oozie/trunk/bin: test-patch
test-patch-00-clean test-patch-20-tests
test-patch-find-new-patch-available-jiras
Author: tucu
Date: Mon Aug 20 17:51:36 2012
New Revision: 1375141
URL: http://svn.apache.org/viewvc?rev=1375141&view=rev
Log:
Improvements to test-patch script (tucu)
Added:
incubator/oozie/trunk/bin/test-patch-00-clean (with props)
incubator/oozie/trunk/bin/test-patch-find-new-patch-available-jiras (with props)
Modified:
incubator/oozie/trunk/bin/test-patch
incubator/oozie/trunk/bin/test-patch-20-tests
Modified: incubator/oozie/trunk/bin/test-patch
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/bin/test-patch?rev=1375141&r1=1375140&r2=1375141&view=diff
==============================================================================
--- incubator/oozie/trunk/bin/test-patch (original)
+++ incubator/oozie/trunk/bin/test-patch Mon Aug 20 17:51:36 2012
@@ -203,7 +203,7 @@ run() {
outputFile="`basename $1`-$2.out"
bash ${DEBUG} $1 --op=$2 --tempdir=${TEMPDIR} --reportdir=${REPORTDIR} \
--summaryfile=${SUMMARYFILE} --patchfile=${PATCHFILE} ${MVNPASSTHRU} \
- ${VERBOSEOPTION} 2>&1 | tee ${TEMPDIR}/${outputFile} >> $STDOUT
+ ${VERBOSEOPTION} | tee ${TEMPDIR}/${outputFile} >> $STDOUT
if [[ $? != 0 ]] ; then
echo " Failure, check for details ${TEMPDIR}/${outputFile}"
echo
Added: incubator/oozie/trunk/bin/test-patch-00-clean
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/bin/test-patch-00-clean?rev=1375141&view=auto
==============================================================================
--- incubator/oozie/trunk/bin/test-patch-00-clean (added)
+++ incubator/oozie/trunk/bin/test-patch-00-clean Mon Aug 20 17:51:36 2012
@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BASEDIR=$(pwd)
+TASKNAME="CLEAN"
+OP=""
+TEMPDIR=""
+REPORTDIR=""
+SUMMARYFILE=""
+MVNPASSTHRU=""
+
+###############################################################################
+cleanupAndExit() {
+ exit $1
+}
+###############################################################################
+printUsage() {
+ echo "Usage: $0 --taskname | (--op=pre|post|report --tempdir=<TEMP DIR>) [-D<VALUE>...] [-P<VALUE>...]"
+ echo
+}
+###############################################################################
+parseArgs() {
+ for i in $*
+ do
+ case $i in
+ --taskname)
+ echo ${TASKNAME}
+ exit 0
+ ;;
+ --op=*)
+ OP=${i#*=}
+ ;;
+ --tempdir=*)
+ TEMPDIR=${i#*=}
+ ;;
+ --reportdir=*)
+ REPORTDIR=${i#*=}
+ ;;
+ --summaryfile=*)
+ SUMMARYFILE=${i#*=}
+ ;;
+ -D*)
+ MVNPASSTHRU="${MVNPASSTHRU} $i"
+ ;;
+ -P*)
+ MVNPASSTHRU="${MVNPASSTHRU} $i"
+ ;;
+ esac
+ done
+ if [[ "${OP}" == "" || "${TEMPDIR}" == "" ]] ; then
+ echo "Missing options"
+ echo
+ printUsage
+ cleanupAndExit 1
+ fi
+ if [[ "${OP}" != "pre" && "${OP}" != "post" && "${OP}" != "report" ]] ; then
+ echo "Invalid operation"
+ echo
+ printUsage
+ cleanupAndExit 1
+ fi
+}
+###############################################################################
+
+
+parseArgs "$@"
+
+case $OP in
+ pre)
+ mvn clean ${MVNPASSTHRU} > ${TEMPDIR}/${TASKNAME}.txt
+ exit $?
+ ;;
+ post)
+ ;;
+ report)
+ echo " ${TASKNAME} cleaned target directories" >> $SUMMARYFILE
+ ;;
+esac
+
+exit 0
Propchange: incubator/oozie/trunk/bin/test-patch-00-clean
------------------------------------------------------------------------------
svn:executable = *
Modified: incubator/oozie/trunk/bin/test-patch-20-tests
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/bin/test-patch-20-tests?rev=1375141&r1=1375140&r2=1375141&view=diff
==============================================================================
--- incubator/oozie/trunk/bin/test-patch-20-tests (original)
+++ incubator/oozie/trunk/bin/test-patch-20-tests Mon Aug 20 17:51:36 2012
@@ -86,17 +86,22 @@ case $OP in
mvn test ${MVNPASSTHRU} -Dmaven.test.failure.ignore=true \
-Dmaven.test.error.ignore=true -fae -Doozie.test.waitfor.ratio=3 \
-Dtest.timeout=7200 | tee ${TEMPDIR}/${TASKNAME}.out >> $STDOUT
- mvn test -Dtest=TestHiveActionExecutor,TestHiveMain ${MVNPASSTHRU} \
- -PtestHive -Dmaven.test.failure.ignore=true -Dmaven.test.error.ignore=true \
- -fae -Doozie.test.waitfor.ratio=3 -Dtest.timeout=7200 \
- | tee -a ${TEMPDIR}/${TASKNAME}.out >> $STDOUT
+
+ if [[ ! "${MVNPASSTHRU}" =~ "-Dtest=" || ("${MVNPASSTHRU}" =~ "TestHiveActionExecutor" || "${MVNPASSTHRU}" =~ "TestHiveMain") ]] ; then
+ # we need to do this 2nd run because Pig 0.9 and Hive 0.9 require different
+ # versions of antlr. Once we move to Pig 0.10 then we don't need this.
+ mvn test -Dtest=TestHiveActionExecutor,TestHiveMain ${MVNPASSTHRU} \
+ -PtestHive -Dmaven.test.failure.ignore=true -Dmaven.test.error.ignore=true \
+ -fae -Doozie.test.waitfor.ratio=3 -Dtest.timeout=7200 \
+ | tee -a ${TEMPDIR}/${TASKNAME}.out >> $STDOUT
+ fi
exit 0
;;
report)
failedTests=`find . -name 'TEST*.xml' | xargs grep -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
- testsRun=`grep "Tests run:" ${TEMPDIR}/${TASKNAME}.out | awk '{print $3}' | sed 's/,//' | awk 'BEGIN {count=0} {count=count+$1} END {print "Tests run: "count}'`
- testsFailed=`grep "Tests run:" ${TEMPDIR}/${TASKNAME}.out | awk '{print $5}' | sed 's/,//' | awk 'BEGIN {count=0} {count=count+$1} END {print "Tests failures: "count}'`
- testsErrors=`grep "Tests run:" ${TEMPDIR}/${TASKNAME}.out | awk '{print $7}' | sed 's/,//' | awk 'BEGIN {count=0} {count=count+$1} END {print "Tests errors: "count}'`
+ testsRun=`grep "Tests run:" ${TEMPDIR}/${TASKNAME}.out | grep -v " Time elapsed:" | awk '{print $3}' | sed 's/,//' | awk 'BEGIN {count=0} {count=count+$1} END {print "Tests run: "count}'`
+ testsFailed=`grep "Tests run:" ${TEMPDIR}/${TASKNAME}.out | grep -v " Time elapsed:" | awk '{print $5}' | sed 's/,//' | awk 'BEGIN {count=0} {count=count+$1} END {print "Tests failures: "count}'`
+ testsErrors=`grep "Tests run:" ${TEMPDIR}/${TASKNAME}.out | grep -v " Time elapsed:" | awk '{print $7}' | sed 's/,//' | awk 'BEGIN {count=0} {count=count+$1} END {print "Tests errors: "count}'`
if [ -n "${failedTests}" ] ; then
echo "-1 ${TASKNAME} - the patch failed the following testcases:" >> $SUMMARYFILE
echo "" >> ${SUMMARYFILE}
Added: incubator/oozie/trunk/bin/test-patch-find-new-patch-available-jiras
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/bin/test-patch-find-new-patch-available-jiras?rev=1375141&view=auto
==============================================================================
--- incubator/oozie/trunk/bin/test-patch-find-new-patch-available-jiras (added)
+++ incubator/oozie/trunk/bin/test-patch-find-new-patch-available-jiras Mon Aug 20 17:51:36 2012
@@ -0,0 +1,108 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BASEDIR=$(pwd)
+TEMPDIR=${BASEDIR}/tmp
+STDOUT="/dev/null"
+
+TOKEN=""
+RUNTESTSFILE=${BASEDIR}/TESTED_PATCHES.txt
+
+printUsage() {
+ echo "Usage: $0 <OPTIONS>"
+ echo " (--token=<OOZIE PRECOMMIT JOB TOKEN>)"
+ echo " [--verbose]"
+ echo " [--script-debug]"
+ echo
+}
+###############################################################################
+parseArgs() {
+ for i in $*
+ do
+ case $i in
+ --token=*)
+ TOKEN=${i#*=}
+ ;;
+ --script-debug)
+ DEBUG="-x"
+ ;;
+ --verbose)
+ VERBOSEOPTION="--verbose"
+ STDOUT="/dev/stdout"
+ ;;
+ *)
+ echo "Invalid option"
+ echo
+ printUsage
+ exit 1
+ ;;
+ esac
+ done
+ if [ "${TOKEN}" == "" ] ; then
+ echo "Token has not been specified"
+ echo
+ printUsage
+ exit 1
+ fi
+}
+###############################################################################
+findAndSubmitAvailablePatches() {
+
+## Grab all the key (issue numbers) and largest attachment id for each item in the XML
+curl "https://issues.apache.org/jira/sr/jira.issueviews:searchrequest-xml/temp/SearchRequest.xml?jqlQuery=project+in+%28OOZIE%29+AND+status+%3D+%22Patch+Available%22+ORDER+BY+updated+DESC&tempMax=1000" > ${TEMPDIR}/patch-availables.xml
+if [ "$?" != "0" ] ; then
+ echo "Could not retrieve available patches from JIRA"
+ exit 1
+fi
+xpath -q -e "//item/key/text() | //item/attachments/attachment[not(../attachment/@id > @id)]/@id" ${TEMPDIR}/patch-availables.xml | tee -a $STDOUT > ${TEMPDIR}/patch-attachments.element
+
+### Replace newlines with nothing, then replace id=" with =, then replace " with newline
+### to yield lines with pairs (issueNumber,largestAttachmentId). Example: OOZIE-123,456984
+cat ${TEMPDIR}/patch-attachments.element | awk '{ if ( $1 ~ /^OOZIE\-/) {JIRA=$1 }; if ($1 ~ /id=/) { print JIRA","$1} }' | sed 's/id\="//' | sed 's/"//' | tee -a $STDOUT > ${TEMPDIR}/patch-availables.pair
+
+### Iterate through issue list and find the (issueNumber,largestAttachmentId) pairs that have
+### not been tested (ie don't already exist in the patch_tested.txt file
+touch ${RUNTESTSFILE}
+cat ${TEMPDIR}/patch-availables.pair | while read PAIR ; do
+ set +e
+ COUNT=`grep -c "$PAIR" ${RUNTESTSFILE}`
+ set -e
+ if [ "$COUNT" -lt "1" ] ; then
+ ### Parse $PAIR into project, issue number, and attachment id
+ ISSUE=`echo $PAIR | sed -e "s/,.*$//"`
+ ### Kick off job
+ echo "Starting job for issue $ISSUE"
+ curl --fail --location --retry 3 "https://builds.apache.org/job/oozie-trunk-precommit-build/buildWithParameters?token=${TOKEN}&JIRA_NUMBER=${ISSUE}" >> $STDOUT
+ if [ "$?" != "0" ] ; then
+ echo "Could not submit precommit job for $ISSUE"
+ exit 1
+ fi
+ ### Mark this pair as tested by appending to file
+ echo "$PAIR" >> ${RUNTESTSFILE}
+ fi
+done
+}
+###############################################################################
+
+mkdir ${TEMPDIR} 2> $STDOUT
+
+parseArgs "$@"
+
+if [ -n "${DEBUG}" ] ; then
+ set -x
+fi
+
+findAndSubmitAvailablePatches
+
+exit 0
Propchange: incubator/oozie/trunk/bin/test-patch-find-new-patch-available-jiras
------------------------------------------------------------------------------
svn:executable = *