You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2017/03/27 15:18:42 UTC
oozie git commit: OOZIE-2841 Limit FindBugs diff errors in JIRA
comments (andras.piros via gezapeti)
Repository: oozie
Updated Branches:
refs/heads/master 1392eeaf8 -> 890f69ead
OOZIE-2841 Limit FindBugs diff errors in JIRA comments (andras.piros via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/890f69ea
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/890f69ea
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/890f69ea
Branch: refs/heads/master
Commit: 890f69ead827467381983b185fd4fa9a6a0780cd
Parents: 1392eea
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Mon Mar 27 17:18:17 2017 +0200
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Mon Mar 27 17:18:17 2017 +0200
----------------------------------------------------------------------
bin/test-patch | 65 +++++++++++++++++------------
bin/test-patch-11-findbugs-diff | 79 +++++++++++++++++++++++++++++-------
release-log.txt | 1 +
3 files changed, 103 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/890f69ea/bin/test-patch
----------------------------------------------------------------------
diff --git a/bin/test-patch b/bin/test-patch
index d69cc59..33a6d11 100755
--- a/bin/test-patch
+++ b/bin/test-patch
@@ -23,6 +23,8 @@ TEMPDIR=${TESTPATCHDIR}/tmp
REPORTDIR=${TESTPATCHDIR}/reports
SUMMARYFILE=${REPORTDIR}/TEST-SUMMARY.jira
SUMMARYFILETXT=${REPORTDIR}/TEST-SUMMARY.txt
+SUMMARYFILE_FULL=${REPORTDIR}/TEST-SUMMARY-FULL.jira
+SUMMARYFILE_FULL_TXT=${REPORTDIR}/TEST-SUMMARY-FULL.txt
JIRAHOST="https://issues.apache.org"
JIRAURL="${JIRAHOST}/jira"
@@ -84,9 +86,9 @@ prepareSCM() {
echo "It should not happen DIRTYSCM=false & RESETSCM=false"
exit 1
fi
- echo "Cleaning local ${SCM} workspace" >> ${SUMMARYFILE}
+ summary_both "Cleaning local ${SCM} workspace"
else
- echo "WARNING: Running test-patch on a dirty local ${SCM} workspace" >> ${SUMMARYFILE}
+ summary_both "WARNING: Running test-patch on a dirty local ${SCM} workspace"
fi
}
###############################################################################
@@ -292,16 +294,16 @@ applyPatch() {
fi
else
echo "Patch failed to apply to head of branch"
- echo "{color:red}-1{color} Patch failed to apply to head of branch" >> ${SUMMARYFILE}
- echo "" >> ${SUMMARYFILE}
- echo "----------------------------" >> ${SUMMARYFILE}
+ summary_both "{color:red}-1{color} Patch failed to apply to head of branch"
+ summary_both ""
+ summary_both "----------------------------"
echo
cleanupAndExit 1
fi
fi
echo "" >> $STDOUT
echo "Patch applied"
- echo "{color:green}+1 PATCH_APPLIES{color}" >> $SUMMARYFILE
+ summary_both "{color:green}+1 PATCH_APPLIES{color}"
echo
}
###############################################################################
@@ -312,7 +314,7 @@ run() {
echo " Running test-patch task ${task}"
outputFile="`basename $1`-$2.out"
$1 --op=$2 --tempdir=${TEMPDIR} --reportdir=${REPORTDIR} \
- --summaryfile=${SUMMARYFILE} --patchfile=${PATCHFILE} ${MVNPASSTHRU} \
+ --summaryfile=${SUMMARYFILE} --summaryfile-full=${SUMMARYFILE_FULL} --patchfile=${PATCHFILE} ${MVNPASSTHRU} \
${VERBOSEOPTION} | tee ${TEMPDIR}/${outputFile} >> $STDOUT
if [[ $? != 0 ]] ; then
echo " Failure, check for details ${TEMPDIR}/${outputFile}"
@@ -351,6 +353,12 @@ createReports() {
echo
}
###############################################################################
+summary_both() {
+ LINE=$1
+ echo ${LINE} >> ${SUMMARYFILE}
+ echo ${LINE} >> ${SUMMARYFILE_FULL}
+}
+###############################################################################
echo
@@ -359,12 +367,13 @@ parseArgs "$@"
prepareTestPatchDirs
echo "" > ${SUMMARYFILE}
+echo "" > ${SUMMARYFILE_FULL}
if [ "${PATCHFILE}" == "" ] ; then
echo "Testing JIRA ${JIRAISSUE}"
echo
- echo "Testing JIRA ${JIRAISSUE}" >> ${SUMMARYFILE}
- echo "" >> ${SUMMARYFILE}
+ summary_both "Testing JIRA ${JIRAISSUE}"
+ summary_both ""
else
if [ ! -e ${PATCHFILE} ] ; then
echo "Patch file does not exist"
@@ -372,54 +381,56 @@ else
fi
echo "Testing patch ${PATCHFILE}"
echo
- echo "Testing patch ${PATCHFILE}" >> ${SUMMARYFILE}
- echo "" >> ${SUMMARYFILE}
+ summary_both "Testing patch ${PATCHFILE}"
+ summary_both ""
fi
prepareSCM
-echo "" >> ${SUMMARYFILE}
+summary_both ""
if [ "${PATCHFILE}" == "" ] ; then
downloadPatch ${JIRAISSUE}
fi
-echo "----------------------------" >> ${SUMMARYFILE}
-echo "" >> ${SUMMARYFILE}
+summary_both "----------------------------"
+summary_both ""
getAllTasks
prePatchRun
applyPatch
postPatchRun
createReports
-echo "" >> ${SUMMARYFILE}
-echo "----------------------------" >> ${SUMMARYFILE}
+summary_both ""
+summary_both "----------------------------"
MINUSONES=`grep -c "\}\-1" ${SUMMARYFILE}`
if [[ $MINUSONES == 0 ]]; then
- echo "{color:green}*+1 Overall result, good!, no -1s*{color}" >> ${SUMMARYFILE}
+ summary_both "{color:green}*+1 Overall result, good!, no -1s*{color}"
else
- echo "{color:red}*-1 Overall result, please check the reported -1(s)*{color}" >> ${SUMMARYFILE}
+ summary_both "{color:red}*-1 Overall result, please check the reported -1(s)*{color}"
fi
-echo "" >> ${SUMMARYFILE}
+summary_both ""
WARNINGS=`grep -c "\}WARNING" ${SUMMARYFILE}`
if [[ $WARNINGS != 0 ]]; then
- echo "{color:red}. There is at least one warning, please check{color}" >> ${SUMMARYFILE}
+ summary_both "{color:red}. There is at least one warning, please check{color}"
fi
-echo "" >> ${SUMMARYFILE}
+summary_both ""
if [ ! -z "${JIRAISSUE}" ]; then
- echo "The full output of the test-patch run is available at" >> ${SUMMARYFILE}
- echo "" >> ${SUMMARYFILE}
- echo ". ${BUILD_URL}" >> ${SUMMARYFILE}
- echo "" >> ${SUMMARYFILE}
+ summary_both "The full output of the test-patch run is available at"
+ summary_both ""
+ summary_both ". ${BUILD_URL}"
+ summary_both ""
else
echo
echo "Refer to ${REPORTDIR} for detailed test-patch reports"
echo
fi
-cat ${SUMMARYFILE} | sed -e 's/{color}//' -e 's/{color:green}//' -e 's/{color:red}//' -e 's/^\.//' -e 's/^\*//' -e 's/\*$//' > ${SUMMARYFILETXT}
-
+cat ${SUMMARYFILE} | sed -e 's/{color}//' -e 's/{color:green}//' -e 's/{color:red}//' -e 's/{color:orange}//' -e 's/^\.//' -e 's/^\*//' -e 's/\*$//' > ${SUMMARYFILETXT}
cat ${SUMMARYFILETXT}
+cat ${SUMMARYFILE_FULL} | sed -e 's/{color}//' -e 's/{color:green}//' -e 's/{color:red}//' -e 's/{color:orange}//' -e 's/^\.//' -e 's/^\*//' -e 's/\*$//' > ${SUMMARYFILE_FULL_TXT}
+
grep "^+1 Overall result" ${SUMMARYFILETXT} &> /dev/null
+grep "^+1 Overall result" ${SUMMARYFILE_FULL_TXT} &> /dev/null
cleanupAndExit "$?"
http://git-wip-us.apache.org/repos/asf/oozie/blob/890f69ea/bin/test-patch-11-findbugs-diff
----------------------------------------------------------------------
diff --git a/bin/test-patch-11-findbugs-diff b/bin/test-patch-11-findbugs-diff
index f7a4b2f..8001bb8 100755
--- a/bin/test-patch-11-findbugs-diff
+++ b/bin/test-patch-11-findbugs-diff
@@ -22,6 +22,7 @@ OP=""
TEMPDIR=${BASEDIR}/tmp
REPORTDIR=""
SUMMARYFILE=""
+SUMMARYFILE_FULL=""
STDOUT="/dev/null"
MVNPASSTHRU=""
FINDBUGS_JAR_URL=https://repo1.maven.org/maven2/me/andrz/findbugs/findbugs-diff/0.1.0/findbugs-diff-0.1.0-all.jar
@@ -31,6 +32,7 @@ FINDBUGS_PRIORITY_THRESHOLD=2
# Scariest and scary
FINDBUGS_RANK_THRESHOLD=9
FINDBUGS_XML_NAME=findbugsXml.xml
+BUG_LIMIT_PER_PROJECT=5
cleanup_and_exit() {
@@ -52,7 +54,7 @@ remove_file_if_present() {
}
print_usage() {
- echo "Usage: $0 --taskname | (--op=pre|post|report --tempdir=<TEMP DIR> --reportdir=<REPORT DIR> --summaryfile=<SUMMARY FILE>) [--verbose] [-D<VALUE>...] [-P<VALUE>...]"
+ echo "Usage: $0 --taskname | (--op=pre|post|report --tempdir=<TEMP DIR> --reportdir=<REPORT DIR> --summaryfile=<SUMMARY FILE> --summaryfile-full=<FULL SUMMARY FILE>) [--verbose] [-D<VALUE>...] [-P<VALUE>...]"
echo
}
@@ -76,6 +78,9 @@ parse_args() {
--summaryfile=*)
SUMMARYFILE=${i#*=}
;;
+ --summaryfile-full=*)
+ SUMMARYFILE_FULL=${i#*=}
+ ;;
--verbose)
STDOUT="/dev/stdout"
;;
@@ -87,7 +92,7 @@ parse_args() {
;;
esac
done
- if [[ "${TASKNAME}" == "" || "${OP}" == "" || "${TEMPDIR}" == "" || "${REPORTDIR}" == "" || "${SUMMARYFILE}" == "" ]] ; then
+ if [[ "${TASKNAME}" == "" || "${OP}" == "" || "${TEMPDIR}" == "" || "${REPORTDIR}" == "" || "${SUMMARYFILE}" == "" || "${SUMMARYFILE_FULL}" == "" ]] ; then
echo "Missing options"
echo
print_usage
@@ -108,9 +113,9 @@ verify_and_save_findbugs_output() {
REPORT_SUFFIX=$3
echo "[TRACE] Verifying and saving FindBugs output in ${BRANCH_LABEL}"
- echo "[TRACE] mvn clean verify -DskipTests ${MVNPASSTHRU} | tee ${REPORTDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt >> ${STDOUT}"
+ echo "[TRACE] mvn clean compile test-compile findbugs:check -Dfindbugs.failOnError=false -Dcheckstyle.failOnViolation=false -DskipTests ${MVNPASSTHRU} | tee ${REPORTDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt >> ${STDOUT}"
- mvn clean verify -DskipTests ${MVNPASSTHRU} | tee ${REPORTDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt >> ${STDOUT}
+ mvn clean compile test-compile findbugs:check -Dfindbugs.failOnError=false -Dcheckstyle.failOnViolation=false -DskipTests ${MVNPASSTHRU} | tee ${REPORTDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt >> ${STDOUT}
if [ ! -d "$FINDBUGS_OUTPUT_DIR" ]; then
mkdir -p "${FINDBUGS_OUTPUT_DIR}"
@@ -182,7 +187,7 @@ download_and_check_findbugs_diff_jar() {
echo "${md5Content}" > "${DIFF_DIR}/${FINDBUGS_JAR}.md5sum"
else
echo "[ERROR] Neither md5 nor md5sum are present, cannot check FindBugs diff JAR"
- echo "{color:red}-1{color} Neither md5 nor md5sum are present, cannot check FindBugs diff JAR." >> "${SUMMARYFILE}"
+ summary_both "{color:red}-1{color} Neither md5 nor md5sum are present, cannot check FindBugs diff JAR."
cleanup_and_exit 1
fi
@@ -200,7 +205,7 @@ download_and_check_findbugs_diff_jar() {
fi
echo "[ERROR] FindBugs diff JAR has a weird MD5 sum, rejecting"
- echo "{color:red}-1{color} FindBugs diff JAR has a weird MD5 sum, rejecting." >> "${SUMMARYFILE}"
+ summary_both "{color:red}-1{color} FindBugs diff JAR has a weird MD5 sum, rejecting."
cleanup_and_exit 1
fi
@@ -257,6 +262,7 @@ check_minimum_file_size() {
check_findbugs_diffs_and_create_reports() {
DIFF_DIR=$1
REPORT=()
+ REPORT_FULL=()
echo "[TRACE] Checking FindBugs diffs and creating reports"
@@ -284,47 +290,88 @@ check_findbugs_diffs_and_create_reports() {
if [ "${newBugBelowThresholdCount}" -gt "0" ]; then
echo "[ERROR] There are [${newBugBelowThresholdCount}] new bugs found below threshold in [${componentDir}]."
- REPORT+=("{color:red}-1{color} There are [${newBugBelowThresholdCount}] new bugs found below threshold in [${componentDir}] that must be fixed.")
+ REPORT_FULL+=("{color:red}-1{color} There are [${newBugBelowThresholdCount}] new bugs found below threshold in [${componentDir}] that must be fixed.")
+ if [ "${newBugBelowThresholdCount}" -gt "${BUG_LIMIT_PER_PROJECT}" ]; then
+ REPORT+=("{color:red}-1{color} There are [${newBugBelowThresholdCount}] new bugs found below threshold in [${componentDir}] that must be fixed, listing only the first [${BUG_LIMIT_PER_PROJECT}] ones.")
+ else
+ REPORT+=("{color:red}-1{color} There are [${newBugBelowThresholdCount}] new bugs found below threshold in [${componentDir}] that must be fixed.")
+ fi
echo "[DEBUG] You can find the FindBugs diff here (look for the red and orange ones): ${htmlFileName}"
REPORT+=("You can find the FindBugs diff here (look for the red and orange ones): ${htmlFileName}")
- REPORT+=("The most important FindBugs errors are:")
+ REPORT_FULL+=("You can find the FindBugs diff here (look for the red and orange ones): ${htmlFileName}")
+ REPORT_FULL+=("The most important FindBugs errors are:")
+ if [ "${newBugBelowThresholdCount}" -gt "${BUG_LIMIT_PER_PROJECT}" ]; then
+ REPORT+=("The top [${BUG_LIMIT_PER_PROJECT}] most important FindBugs errors are:")
+ else
+ REPORT+=("The most important FindBugs errors are:")
+ fi
+
+ lineCount=0
while IFS= read -r line; do
- REPORT+=( "${line}" );
+ REPORT_FULL+=( "${line}" );
+ if [ "${lineCount}" -lt "${BUG_LIMIT_PER_PROJECT}" ]; then
+ REPORT+=( "${line}" );
+ fi
+ lineCount=$((lineCount + 1))
done < <(echo cat "/BugCollection/BugInstance[@priority <= ${FINDBUGS_PRIORITY_THRESHOLD} or @rank <= ${FINDBUGS_RANK_THRESHOLD}]/SourceLine/Message/text() | /BugCollection/BugInstance[@priority <= 2 or @rank <= 9]/LongMessage/text()" \
| xmllint --shell "${fn}" | grep -v '\-\-\-\-\-\-\-' | grep -v '/ >' | sed -e 'N;s/\(.*\)\n\(.*\)/\2: \1/')
elif [ "${newBugTotalCount}" -gt "0" ]; then
echo "[WARN] There are [${newBugTotalCount}] new bugs found in [${componentDir}]."
REPORT+=("{color:orange}0{color} There are [${newBugTotalCount}] new bugs found in [${componentDir}] that would be nice to have fixed.")
+ REPORT_FULL+=("{color:orange}0{color} There are [${newBugTotalCount}] new bugs found in [${componentDir}] that would be nice to have fixed.")
echo "[DEBUG] You can find the FindBugs diff here: ${htmlFileName}"
REPORT+=("You can find the FindBugs diff here: ${htmlFileName}")
+ REPORT_FULL+=("You can find the FindBugs diff here: ${htmlFileName}")
else
echo "[DEBUG] There are no new bugs found in [${componentDir}]."
REPORT+=("{color:green}+1{color} There are no new bugs found in [${componentDir}].")
+ REPORT_FULL+=("{color:green}+1{color} There are no new bugs found in [${componentDir}].")
fi
done < <(find "${DIFF_DIR}" -name findbugs-new.xml -print0)
if [ "${belowThresholdCount}" -gt "0" ]; then
echo "[ERROR] There are [${belowThresholdCount}] new bugs found below threshold in total that must be fixed."
- echo "{color:red}-1{color} There are [${belowThresholdCount}] new bugs found below threshold in total that must be fixed." \
- >> "${SUMMARYFILE}"
+ summary_both "{color:red}-1{color} There are [${belowThresholdCount}] new bugs found below threshold in total that must be fixed."
elif [ "${totalCount}" -gt "0" ]; then
echo "[WARN] There are [${totalCount}] new bugs found in total that would be nice to have fixed."
- echo "{color:orange}0{color} There are [${totalCount}] new bugs found in total that would be nice to have fixed." \
- >> "${SUMMARYFILE}"
+ summary_both "{color:orange}0{color} There are [${totalCount}] new bugs found in total that would be nice to have fixed."
else
echo "[INFO] There are no new bugs found totally]."
- echo "{color:green}+1{color} There are no new bugs found in total." >> "${SUMMARYFILE}"
+ summary_both "{color:green}+1{color} There are no new bugs found in total."
fi
for line in "${REPORT[@]}" ; do
- echo ". ${line}" >> "${SUMMARYFILE}"
+ summary ". ${line}"
+ done
+
+ for line in "${REPORT_FULL[@]}" ; do
+ summary_full ". ${line}"
done
echo "[TRACE] FindBugs diffs checked and reports created"
}
+summary() {
+ LINE=$1
+ echo ${LINE} >> ${SUMMARYFILE}
+}
+
+
+summary_full() {
+ LINE=$1
+ echo ${LINE} >> ${SUMMARYFILE_FULL}
+}
+
+
+summary_both() {
+ LINE=$1
+ echo ${LINE} >> ${SUMMARYFILE}
+ echo ${LINE} >> ${SUMMARYFILE_FULL}
+}
+
+
parse_args "$@"
case ${OP} in
@@ -339,6 +386,8 @@ case ${OP} in
download_and_check_findbugs_diff_jar "${TEMPDIR}/${TASKNAME}/diff"
perform_findbugs_diffs "${TEMPDIR}/${TASKNAME}/pre" "${TEMPDIR}/${TASKNAME}/post" "${TEMPDIR}/${TASKNAME}/diff"
check_findbugs_diffs_and_create_reports "${TEMPDIR}/${TASKNAME}/diff"
+ echo "[TRACE] Summary file size is $(wc -c ${SUMMARYFILE} | awk '{print $1}') bytes"
+ echo "[TRACE] Full summary file size is $(wc -c ${SUMMARYFILE_FULL} | awk '{print $1}') bytes"
;;
esac
http://git-wip-us.apache.org/repos/asf/oozie/blob/890f69ea/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f9b926b..17c9895 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2841 Limit FindBugs diff errors in JIRA comments
OOZIE-2739 Remove property expansion pattern from ShellMain's log4j properties content (harsh)
OOZIE-2817 amend Increase test case stability in pre-commit job (abhishekbafna)
OOZIE-2838 TestClassUtils,TestJsonUtils,TestWritableUtils,TestXmlUtils shall not be an Oozie XTestCase (asasvari via pbacsko)