You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2016/10/19 01:28:54 UTC
oozie git commit: OOZIE-1986 Add FindBugs report to pre-commit build
(andras.piros via rkanter)
Repository: oozie
Updated Branches:
refs/heads/master bd5445280 -> 51920be84
OOZIE-1986 Add FindBugs report to pre-commit build (andras.piros via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/51920be8
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/51920be8
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/51920be8
Branch: refs/heads/master
Commit: 51920be8480c80638e11faff20ad089b02a7f5fb
Parents: bd54452
Author: Robert Kanter <rk...@cloudera.com>
Authored: Tue Oct 18 18:27:20 2016 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Tue Oct 18 18:27:20 2016 -0700
----------------------------------------------------------------------
bin/test-patch-11-findbugs-diff | 274 +++++++++++++++++++
...est-patch-11-findbugs-diff-0.1.0-all.jar.md5 | 1 +
pom.xml | 1 +
release-log.txt | 1 +
4 files changed, 277 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/bin/test-patch-11-findbugs-diff
----------------------------------------------------------------------
diff --git a/bin/test-patch-11-findbugs-diff b/bin/test-patch-11-findbugs-diff
new file mode 100644
index 0000000..0376ab0
--- /dev/null
+++ b/bin/test-patch-11-findbugs-diff
@@ -0,0 +1,274 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "${TESTPATCHDEBUG}" == "true" ] ; then
+ set -x
+fi
+
+BASEDIR=$(pwd)
+TASKNAME="FINDBUGS_DIFF"
+OP=""
+TEMPDIR=${BASEDIR}/tmp
+REPORTDIR=""
+SUMMARYFILE=""
+STDOUT="/dev/null"
+MVNPASSTHRU=""
+FINDBUGS_JAR_URL=https://repo1.maven.org/maven2/me/andrz/findbugs/findbugs-diff/0.1.0/findbugs-diff-0.1.0-all.jar
+FINDBUGS_JAR=findbugs-diff-0.1.0-all.jar
+# Priorities.HIGH_PRIORITY and Priorities.NORMAL_PRIORITY
+FINDBUGS_PRIORITY_THRESHOLD=2
+# Scariest and scary
+FINDBUGS_RANK_THRESHOLD=9
+FINDBUGS_XML_NAME=findbugsXml.xml
+
+
+cleanup_and_exit() {
+ remove_file_if_present "${DIFF_DIR}/${FINDBUGS_JAR}"
+ remove_file_if_present "${DIFF_DIR}/${FINDBUGS_JAR}.md5"
+
+ exit "$1"
+}
+
+
+remove_file_if_present() {
+ FILE_NAME=$1
+
+ if [ -f "${FILE_NAME}" ]; then
+ rm -f "${FILE_NAME}"
+ echo "[TRACE] File [${FILE_NAME}] removed"
+ fi
+}
+
+print_usage() {
+ echo "Usage: $0 --taskname | (--op=pre|post|report --tempdir=<TEMP DIR> --reportdir=<REPORT DIR> --summaryfile=<SUMMARY FILE>) [--verbose] [-D<VALUE>...] [-P<VALUE>...]"
+ echo
+}
+
+
+parse_args() {
+ for i in "$@"; do
+ case $i in
+ --taskname)
+ echo ${TASKNAME}
+ exit 0
+ ;;
+ --op=*)
+ OP=${i#*=}
+ ;;
+ --tempdir=*)
+ TEMPDIR=${i#*=}
+ ;;
+ --reportdir=*)
+ REPORTDIR=${i#*=}
+ ;;
+ --summaryfile=*)
+ SUMMARYFILE=${i#*=}
+ ;;
+ --verbose)
+ STDOUT="/dev/stdout"
+ ;;
+ -D*)
+ MVNPASSTHRU="${MVNPASSTHRU} $i"
+ ;;
+ -P*)
+ MVNPASSTHRU="${MVNPASSTHRU} $i"
+ ;;
+ esac
+ done
+ if [[ "${TASKNAME}" == "" || "${OP}" == "" || "${TEMPDIR}" == "" || "${REPORTDIR}" == "" || "${SUMMARYFILE}" == "" ]] ; then
+ echo "Missing options"
+ echo
+ print_usage
+ cleanup_and_exit 1
+ fi
+ if [[ "${OP}" != "pre" && "${OP}" != "post" && "${OP}" != "report" ]] ; then
+ echo "Invalid operation"
+ echo
+ print_usage
+ cleanup_and_exit 1
+ fi
+}
+
+
+verify_and_save_findbugs_output() {
+ FINDBUGS_OUTPUT_DIR=$1
+ BRANCH_LABEL=$2
+ REPORT_SUFFIX=$3
+
+ echo "[TRACE] Verifying and saving FindBugs output in ${BRANCH_LABEL}"
+
+ mvn clean verify -DskipTests "${MVNPASSTHRU}" | tee "${REPORTDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt" >> ${STDOUT}
+
+ if [ ! -d "$FINDBUGS_OUTPUT_DIR" ]; then
+ mkdir -p "${FINDBUGS_OUTPUT_DIR}"
+ fi
+
+ find . -name ${FINDBUGS_XML_NAME} -exec rsync -avqR {} "${FINDBUGS_OUTPUT_DIR}" ";"
+
+ if [ "$(ls -A "${FINDBUGS_OUTPUT_DIR}")" ] ; then
+ echo "{color:green}+1{color} ${BRANCH_LABEL} produces FindBugs output" >> "${TEMPDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt"
+ else
+ echo "{color:red}-1{color} ${BRANCH_LABEL} does not produce FindBugs output" >> "${TEMPDIR}/${TASKNAME}-${REPORT_SUFFIX}.txt"
+ fi
+
+ echo "[TRACE] FindBugs output in ${BRANCH_LABEL} verified and saved"
+}
+
+
+download_and_check_findbugs_diff_jar() {
+ DIFF_DIR=$1
+ BASH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+ mkdir -p "${DIFF_DIR}"
+
+ echo "[TRACE] Downloading FindBugs diff JAR from ${FINDBUGS_JAR_URL}"
+
+ curl -Ls ${FINDBUGS_JAR_URL} > "${DIFF_DIR}/${FINDBUGS_JAR}"
+
+ echo "[TRACE] FindBugs diff JAR downloaded"
+
+ if hash md5 2>/dev/null; then
+ md5 -q "${DIFF_DIR}/${FINDBUGS_JAR}" > "${DIFF_DIR}/${FINDBUGS_JAR}.md5"
+ elif hash md5sum 2>/dev/null; then
+ md5sum "${DIFF_DIR}/${FINDBUGS_JAR}" > "${DIFF_DIR}/${FINDBUGS_JAR}.md5"
+ else
+ echo "[ERROR] Neither md5 nor md5sum are present, cannot check FindBugs diff JAR"
+ echo "{color:red}-1{color} Neither md5 nor md5sum are present, cannot check FindBugs diff JAR." >> "${SUMMARYFILE}"
+ cleanup_and_exit 1
+ fi
+
+ jarMd5DiffCount=$(grep -Fxvf "${BASH_DIR}/test-patch-11-${FINDBUGS_JAR}.md5" "${DIFF_DIR}/${FINDBUGS_JAR}.md5" | wc -l)
+
+ if [ ${jarMd5DiffCount} -gt "0" ]; then
+ echo "[ERROR] FindBugs diff JAR has a weird MD5 sum, rejecting"
+ echo "{color:red}-1{color} FindBugs diff JAR has a weird MD5 sum, rejecting." >> "${SUMMARYFILE}"
+ cleanup_and_exit 1
+ fi
+
+ echo "[TRACE] FindBugs diff JAR checked, is safe to use"
+
+}
+
+
+perform_findbugs_diffs() {
+ PRE_DIR=$1
+ POST_DIR=$2
+ DIFF_DIR=$3
+
+ echo "[TRACE] Performing FindBugs diffs"
+
+ preFindbugsDiffs=()
+ while IFS= read -r -d '' fn; do
+ preFindbugsDiffs+=(${fn})
+ done < <(find "${PRE_DIR}" -name ${FINDBUGS_XML_NAME} -print0)
+
+ postFindbugsDiffs=()
+ while IFS= read -r -d '' fn; do
+ postFindbugsDiffs+=(${fn})
+ done < <(find "${POST_DIR}" -name ${FINDBUGS_XML_NAME} -print0)
+
+ for index in "${!preFindbugsDiffs[@]}"; do
+ preFindbugsDir=${preFindbugsDiffs[index]}
+ diffDirPostfix=${preFindbugsDir/*${TASKNAME}/${TASKNAME}}
+ diffDirPostfix=${diffDirPostfix:18}
+ diffDirPostfix=${diffDirPostfix%%/target/findbugs/findbugsXml.xml}
+
+ java -jar -Dorg.slf4j.simpleLogger.defaultLogLevel=ERROR "${DIFF_DIR}/${FINDBUGS_JAR}" \
+ --outDir="${DIFF_DIR}/${diffDirPostfix}" "${preFindbugsDiffs[index]}" "${postFindbugsDiffs[index]}"
+ done
+
+ echo "[TRACE] FindBugs diffs performed"
+}
+
+
+check_findbugs_diffs_and_create_reports() {
+ DIFF_DIR=$1
+ REPORT=()
+
+ echo "[TRACE] Checking FindBugs diffs and creating reports"
+
+ belowThresholdCount=0
+ totalCount=0
+ while IFS= read -r -d '' fn; do
+ componentDir=${fn/*${TASKNAME}/${TASKNAME}}
+ componentDir=${componentDir:19}
+ htmlFileName=${componentDir%%.xml}.html
+ componentDir=${componentDir%%/findbugs-new.xml}
+
+ newBugTotalCount=$(xmllint --xpath "count(/BugCollection/BugInstance)" "${fn}")
+ newBugBelowThresholdCount=$(xmllint --xpath "count(/BugCollection/BugInstance[@priority <= ${FINDBUGS_PRIORITY_THRESHOLD} or @rank <= ${FINDBUGS_RANK_THRESHOLD}])" "${fn}")
+
+ belowThresholdCount=$((belowThresholdCount + newBugBelowThresholdCount))
+ totalCount=$((totalCount + newBugTotalCount))
+
+ if [ "${newBugBelowThresholdCount}" -gt "0" ]; then
+ echo "[ERROR] There are [${newBugBelowThresholdCount}] new bugs found below threshold in [${componentDir}]."
+ REPORT+=("{color:red}-1{color} There are [${newBugBelowThresholdCount}] new bugs found below threshold in [${componentDir}] that must be fixed.")
+ echo "[DEBUG] You can find the FindBugs diff here (look for the red and orange ones): ${htmlFileName}"
+ REPORT+=("You can find the FindBugs diff here (look for the red and orange ones): ${htmlFileName}")
+ REPORT+=("The most important FindBugs errors are:")
+ while IFS= read -r line; do
+ REPORT+=( "${line}" );
+ done < <(echo cat "/BugCollection/BugInstance[@priority <= ${FINDBUGS_PRIORITY_THRESHOLD} or @rank <= ${FINDBUGS_RANK_THRESHOLD}]/SourceLine/Message/text() | /BugCollection/BugInstance[@priority <= 2 or @rank <= 9]/LongMessage/text()" \
+ | xmllint --shell "${fn}" | grep -v '\-\-\-\-\-\-\-' | grep -v '/ >' | sed -e 'N;s/\(.*\)\n\(.*\)/\2: \1/')
+ elif [ "${newBugTotalCount}" -gt "0" ]; then
+ echo "[WARN] There are [${newBugTotalCount}] new bugs found in [${componentDir}]."
+ REPORT+=("{color:orange}0{color} There are [${newBugTotalCount}] new bugs found in [${componentDir}] that would be nice to have fixed.")
+ echo "[DEBUG] You can find the FindBugs diff here: ${htmlFileName}"
+ REPORT+=("You can find the FindBugs diff here: ${htmlFileName}")
+ else
+ echo "[DEBUG] There are no new bugs found in [${componentDir}]."
+ REPORT+=("{color:green}+1{color} There are no new bugs found in [${componentDir}].")
+ fi
+
+ done < <(find "${DIFF_DIR}" -name findbugs-new.xml -print0)
+
+ if [ "${belowThresholdCount}" -gt "0" ]; then
+ echo "[ERROR] There are [${belowThresholdCount}] new bugs found below threshold in total that must be fixed."
+ echo "{color:red}-1{color} There are [${belowThresholdCount}] new bugs found below threshold in total that must be fixed." \
+ >> "${SUMMARYFILE}"
+ elif [ "${totalCount}" -gt "0" ]; then
+ echo "[WARN] There are [${totalCount}] new bugs found in total that would be nice to have fixed."
+ echo "{color:orange}0{color} There are [${totalCount}] new bugs found in total that would be nice to have fixed." \
+ >> "${SUMMARYFILE}"
+ else
+ echo "[INFO] There are no new bugs found totally]."
+ echo "{color:green}+1{color} There are no new bugs found in total." >> "${SUMMARYFILE}"
+ fi
+
+ for line in "${REPORT[@]}" ; do
+ echo ". ${line}" >> "${SUMMARYFILE}"
+ done
+
+ echo "[TRACE] FindBugs diffs checked and reports created"
+}
+
+
+parse_args "$@"
+
+case ${OP} in
+ pre)
+ verify_and_save_findbugs_output "${TEMPDIR}/${TASKNAME}/pre" HEAD clean
+ ;;
+ post)
+ verify_and_save_findbugs_output "${TEMPDIR}/${TASKNAME}/post" PATCH patch
+ ;;
+ report)
+ download_and_check_findbugs_diff_jar "${TEMPDIR}/${TASKNAME}/diff"
+ perform_findbugs_diffs "${TEMPDIR}/${TASKNAME}/pre" "${TEMPDIR}/${TASKNAME}/post" "${TEMPDIR}/${TASKNAME}/diff"
+ check_findbugs_diffs_and_create_reports "${TEMPDIR}/${TASKNAME}/diff"
+ ;;
+esac
+
+cleanup_and_exit 0
http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5
----------------------------------------------------------------------
diff --git a/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5 b/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5
new file mode 100644
index 0000000..28179b6
--- /dev/null
+++ b/bin/test-patch-11-findbugs-diff-0.1.0-all.jar.md5
@@ -0,0 +1 @@
+d965fab3dbf678189924ad6d61fc7a3d
http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 83a1d42..fcd023c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1826,6 +1826,7 @@
<exclude>**/.idea/**</exclude>
<exclude>*.patch</exclude>
<exclude>**/*.json</exclude>
+ <exclude>bin/test-patch-*</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/oozie/blob/51920be8/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index f73fbb8..855c01d 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-1986 Add FindBugs report to pre-commit build (andras.piros via rkanter)
OOZIE-2634 Queue dump command message is confusing when the queue is empty (andras.piros via rkanter)