You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/02/04 11:13:03 UTC
incubator-atlas git commit: ATLAS-468: added automatic patch
validation scripts
Repository: incubator-atlas
Updated Branches:
refs/heads/master 20608f022 -> bd47628fe
ATLAS-468: added automatic patch validation scripts
Signed-off-by: Shwetha GS <ss...@hortonworks.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/bd47628f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/bd47628f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/bd47628f
Branch: refs/heads/master
Commit: bd47628fef161af96462c25a3f382002c9cb8c34
Parents: 20608f0
Author: Selvamohan Neethiraj <sn...@apache.org>
Authored: Wed Feb 3 23:49:38 2016 -0500
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Thu Feb 4 15:22:08 2016 +0530
----------------------------------------------------------------------
dev-support/findbugsIncludeFile.xml | 25 +
dev-support/smart-apply-patch.sh | 145 ++++++
dev-support/test-patch.sh | 853 +++++++++++++++++++++++++++++++
pom.xml | 3 +-
4 files changed, 1025 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bd47628f/dev-support/findbugsIncludeFile.xml
----------------------------------------------------------------------
diff --git a/dev-support/findbugsIncludeFile.xml b/dev-support/findbugsIncludeFile.xml
new file mode 100644
index 0000000..8623906
--- /dev/null
+++ b/dev-support/findbugsIncludeFile.xml
@@ -0,0 +1,25 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <!--
+ For now, lets find only critical bugs from static code analyzer
+ -->
+ <Match>
+ <Bug Rank="1" />
+ </Match>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bd47628f/dev-support/smart-apply-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
new file mode 100755
index 0000000..2a9dbd2
--- /dev/null
+++ b/dev-support/smart-apply-patch.sh
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+#
+# Determine if the patch file is a git diff file with prefixes.
+# These files are generated via "git diff" *without* the --no-prefix option.
+#
+# We can apply these patches more easily because we know that the a/ and b/
+# prefixes in the "diff" lines stands for the project root directory.
+# So we don't have to hunt for the project root.
+# And of course, we know that the patch file was generated using git, so we
+# know git apply can handle it properly.
+#
+# Arguments: file name.
+# Return: 0 if it is a git diff; 1 otherwise.
+#
+is_git_diff_with_prefix() {
+ DIFF_TYPE="unknown"
+ while read -r line; do
+ if [[ "$line" =~ ^diff\ ]]; then
+ if [[ "$line" =~ ^diff\ \-\-git ]]; then
+ DIFF_TYPE="git"
+ else
+ return 1 # All diff lines must be diff --git lines.
+ fi
+ fi
+ if [[ "$line" =~ ^\+\+\+\ ]] ||
+ [[ "$line" =~ ^\-\-\-\ ]]; then
+ if ! [[ "$line" =~ ^....[ab]/ ]]; then
+ return 1 # All +++ and --- lines must start with a/ or b/.
+ fi
+ fi
+ done < $1
+ [ x$DIFF_TYPE == x"git" ] || return 1
+ return 0 # return true (= 0 in bash)
+}
+
+PATCH_FILE=$1
+DRY_RUN=$2
+if [ -z "$PATCH_FILE" ]; then
+ echo usage: $0 patch-file
+ exit 1
+fi
+
+PATCH=${PATCH:-patch} # allow overriding patch binary
+
+# Cleanup handler for temporary files
+TOCLEAN=""
+cleanup() {
+ rm $TOCLEAN
+ exit $1
+}
+trap "cleanup 1" HUP INT QUIT TERM
+
+# Allow passing "-" for stdin patches
+if [ "$PATCH_FILE" == "-" ]; then
+ PATCH_FILE=/tmp/tmp.in.$$
+ cat /dev/fd/0 > $PATCH_FILE
+ TOCLEAN="$TOCLEAN $PATCH_FILE"
+fi
+
+# Special case for git-diff patches without --no-prefix
+if is_git_diff_with_prefix "$PATCH_FILE"; then
+ GIT_FLAGS="--binary -p1 -v"
+ if [[ -z $DRY_RUN ]]; then
+ GIT_FLAGS="$GIT_FLAGS --stat --apply "
+ echo Going to apply git patch with: git apply "${GIT_FLAGS}"
+ else
+ GIT_FLAGS="$GIT_FLAGS --check "
+ fi
+ git apply ${GIT_FLAGS} "${PATCH_FILE}"
+ exit $?
+fi
+
+# Come up with a list of changed files into $TMP
+TMP=/tmp/tmp.paths.$$
+TOCLEAN="$TOCLEAN $TMP"
+
+if $PATCH -p0 -E --dry-run < $PATCH_FILE 2>&1 > $TMP; then
+ PLEVEL=0
+ #if the patch applied at P0 there is the possability that all we are doing
+ # is adding new files and they would apply anywhere. So try to guess the
+ # correct place to put those files.
+
+ TMP2=/tmp/tmp.paths.2.$$
+ TOCLEAN="$TOCLEAN $TMP2"
+
+ egrep '^patching file |^checking file ' $TMP | awk '{print $3}' | grep -v /dev/null | sort | uniq > $TMP2
+
+ if [ ! -s $TMP2 ]; then
+ echo "Error: Patch dryrun couldn't detect changes the patch would make. Exiting."
+ cleanup 1
+ fi
+
+ #first off check that all of the files do not exist
+ FOUND_ANY=0
+ for CHECK_FILE in $(cat $TMP2)
+ do
+ if [[ -f $CHECK_FILE ]]; then
+ FOUND_ANY=1
+ fi
+ done
+
+ if [[ "$FOUND_ANY" = "0" ]]; then
+ #all of the files are new files so we have to guess where the correct place to put it is.
+
+ # if all of the lines start with a/ or b/, then this is a git patch that
+ # was generated without --no-prefix
+ if ! grep -qv '^a/\|^b/' $TMP2 ; then
+ echo Looks like this is a git patch. Stripping a/ and b/ prefixes
+ echo and incrementing PLEVEL
+ PLEVEL=$[$PLEVEL + 1]
+ sed -i -e 's,^[ab]/,,' $TMP2
+ fi
+ fi
+elif $PATCH -p1 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+ PLEVEL=1
+elif $PATCH -p2 -E --dry-run < $PATCH_FILE 2>&1 > /dev/null; then
+ PLEVEL=2
+else
+ echo "The patch does not appear to apply with p0 to p2";
+ cleanup 1;
+fi
+
+# If this is a dry run then exit instead of applying the patch
+if [[ -n $DRY_RUN ]]; then
+ cleanup 0;
+fi
+
+echo Going to apply patch with: $PATCH -p$PLEVEL
+$PATCH -p$PLEVEL -E < $PATCH_FILE
+
+cleanup $?
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bd47628f/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
new file mode 100755
index 0000000..9e63575
--- /dev/null
+++ b/dev-support/test-patch.sh
@@ -0,0 +1,853 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#set -x
+ulimit -n 1024
+
+### Setup some variables.
+### BUILD_URL is set by Hudson if it is run by patch process
+### Read variables from properties file
+bindir=$(dirname $0)
+
+# Defaults
+if [ -z "$MAVEN_HOME" ]; then
+ MVN=mvn
+else
+ MVN=$MAVEN_HOME/bin/mvn
+fi
+
+PROJECT_NAME=Atlas
+JENKINS=false
+PATCH_DIR=/tmp
+BASEDIR=$(pwd)
+PS=${PS:-ps}
+AWK=${AWK:-awk}
+WGET=${WGET:-wget}
+GIT=${GIT:-git}
+GREP=${GREP:-grep}
+PATCH=${PATCH:-patch}
+DIFF=${DIFF:-diff}
+JIRACLI=${JIRA:-jira}
+FINDBUGS_HOME=${FINDBUGS_HOME}
+
+###############################################################################
+printUsage() {
+ echo "Usage: $0 [options] patch-file | defect-number"
+ echo
+ echo "Where:"
+ echo " patch-file is a local patch file containing the changes to test"
+ echo " defect-number is a JIRA defect number (e.g. 'ATLAS-1234') to test (Jenkins only)"
+ echo
+ echo "Options:"
+ echo "--patch-dir=<dir> The directory for working and output files (default '/tmp')"
+ echo "--basedir=<dir> The directory to apply the patch to (default current directory)"
+ echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \$MAVEN_HOME/bin/mvn, or 'mvn')"
+ echo "--ps-cmd=<cmd> The 'ps' command to use (default 'ps')"
+ echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
+ echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
+ echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
+ echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
+ echo "--diff-cmd=<cmd> The 'diff' command to use (default 'diff')"
+ echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
+ echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
+ echo "--run-tests Run all tests below the base directory"
+ echo
+ echo "Jenkins-only options:"
+ echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
+ echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
+ echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
+ echo "--jira-password=<pw> The password for the 'jira' command"
+}
+
+###############################################################################
+parseArgs() {
+ for i in $*
+ do
+ case $i in
+ --jenkins)
+ JENKINS=true
+ ;;
+ --patch-dir=*)
+ PATCH_DIR=${i#*=}
+ ;;
+ --basedir=*)
+ BASEDIR=${i#*=}
+ ;;
+ --mvn-cmd=*)
+ MVN=${i#*=}
+ ;;
+ --ps-cmd=*)
+ PS=${i#*=}
+ ;;
+ --awk-cmd=*)
+ AWK=${i#*=}
+ ;;
+ --wget-cmd=*)
+ WGET=${i#*=}
+ ;;
+ --git-cmd=*)
+ GIT=${i#*=}
+ ;;
+ --grep-cmd=*)
+ GREP=${i#*=}
+ ;;
+ --patch-cmd=*)
+ PATCH=${i#*=}
+ ;;
+ --diff-cmd=*)
+ DIFF=${i#*=}
+ ;;
+ --jira-cmd=*)
+ JIRACLI=${i#*=}
+ ;;
+ --jira-password=*)
+ JIRA_PASSWD=${i#*=}
+ ;;
+ --findbugs-home=*)
+ FINDBUGS_HOME=${i#*=}
+ ;;
+ --dirty-workspace)
+ DIRTY_WORKSPACE=true
+ ;;
+ --run-tests)
+ RUN_TESTS=true
+ ;;
+ *)
+ PATCH_OR_DEFECT=$i
+ ;;
+ esac
+ done
+ if [ -z "$PATCH_OR_DEFECT" ]; then
+ printUsage
+ exit 1
+ fi
+ if [[ $JENKINS == "true" ]] ; then
+ echo "Running in Jenkins mode"
+ defect=$PATCH_OR_DEFECT
+ else
+ echo "Running in developer mode"
+ JENKINS=false
+ ### PATCH_FILE contains the location of the patchfile
+ PATCH_FILE=$PATCH_OR_DEFECT
+ if [[ ! -e "$PATCH_FILE" ]] ; then
+ echo "Unable to locate the patch file $PATCH_FILE"
+ cleanupAndExit 0
+ fi
+ ### Check if $PATCH_DIR exists. If it does not exist, create a new directory
+ if [[ ! -e "$PATCH_DIR" ]] ; then
+ mkdir "$PATCH_DIR"
+ if [[ $? == 0 ]] ; then
+ echo "$PATCH_DIR has been created"
+ else
+ echo "Unable to create $PATCH_DIR"
+ cleanupAndExit 0
+ fi
+ fi
+ ### Obtain the patch filename to append it to the version number
+ defect=`basename $PATCH_FILE`
+ fi
+}
+
+###############################################################################
+checkout () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Testing patch for ${defect}."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ ### When run by a developer, if the workspace contains modifications, do not continue
+ ### unless the --dirty-workspace option was set
+ status=`$GIT status --porcelain`
+ if [[ $JENKINS == "false" ]] ; then
+ if [[ "$status" != "" && -z $DIRTY_WORKSPACE ]] ; then
+ echo "ERROR: can't run in a workspace that contains the following modifications"
+ echo "$status"
+ cleanupAndExit 1
+ fi
+ echo
+ else
+ cd $BASEDIR
+ $GIT reset --hard
+ $GIT clean -xdf
+ $GIT checkout master
+ $GIT pull --rebase
+ fi
+ GIT_REVISION=`git rev-parse --verify --short HEAD`
+ return $?
+}
+
+###############################################################################
+downloadPatch () {
+ ### Download latest patch file (ignoring .htm and .html) when run from patch process
+ if [[ $JENKINS == "true" ]] ; then
+ $WGET -q -O $PATCH_DIR/jira http://issues.apache.org/jira/browse/$defect
+ if [[ `$GREP -c 'Patch Available' $PATCH_DIR/jira` == 0 ]] ; then
+ echo "$defect is not \"Patch Available\". Exiting."
+ cleanupAndExit 0
+ fi
+ relativePatchURL=`$GREP -o '"/jira/secure/attachment/[0-9]*/[^"]*' $PATCH_DIR/jira | $GREP -v -e 'htm[l]*$' | sort | tail -1 | $GREP -o '/jira/secure/attachment/[0-9]*/[^"]*'`
+ patchURL="http://issues.apache.org${relativePatchURL}"
+ patchNum=`echo $patchURL | $GREP -o '[0-9]*/' | $GREP -o '[0-9]*'`
+ echo "$defect patch is being downloaded at `date` from"
+ echo "$patchURL"
+ $WGET -q -O $PATCH_DIR/patch $patchURL
+ VERSION=${GIT_REVISION}_${defect}_PATCH-${patchNum}
+ JIRA_COMMENT="Here are the results of testing the latest attachment
+ $patchURL
+ against master revision ${GIT_REVISION}."
+
+ ### Copy the patch file to $PATCH_DIR
+ else
+ VERSION=PATCH-${defect}
+ cp $PATCH_FILE $PATCH_DIR/patch
+ if [[ $? == 0 ]] ; then
+ echo "Patch file $PATCH_FILE copied to $PATCH_DIR"
+ else
+ echo "Could not copy $PATCH_FILE to $PATCH_DIR"
+ cleanupAndExit 0
+ fi
+ fi
+}
+
+###############################################################################
+verifyPatch () {
+ # Before building, check to make sure that the patch is valid
+ $bindir/smart-apply-patch.sh $PATCH_DIR/patch dryrun
+ if [[ $? != 0 ]] ; then
+ echo "PATCH APPLICATION FAILED"
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 patch{color}. The patch command could not apply the patch."
+ return 1
+ else
+ return 0
+ fi
+}
+
+###############################################################################
+prebuildWithoutPatch () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Pre-build master to verify master stability and javac warnings"
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "Compiling $(pwd)"
+ echo "$MVN clean test -DskipTests > $PATCH_DIR/masterJavacWarnings.txt 2>&1"
+ $MVN clean test -DskipTests > $PATCH_DIR/masterJavacWarnings.txt 2>&1
+ if [[ $? != 0 ]] ; then
+ echo "master compilation is broken?"
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 patch{color}. master compilation may be broken."
+ return 1
+ fi
+
+ echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1"
+ $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/masterJavadocWarnings.txt 2>&1
+ if [[ $? != 0 ]] ; then
+ echo "master javadoc compilation is broken?"
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 patch{color}. master compilation may be broken."
+ return 1
+ fi
+
+ return 0
+}
+
+###############################################################################
+### Check for @author tags in the patch
+checkAuthor () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Checking there are no @author tags in the patch."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ authorTags=`$GREP -c -i '@author' $PATCH_DIR/patch`
+ echo "There appear to be $authorTags @author tags in the patch."
+ if [[ $authorTags != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 @author{color}. The patch appears to contain $authorTags @author tags which the Atlas community has agreed to not allow in code contributions."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+1 @author{color}. The patch does not contain any @author tags."
+ return 0
+}
+
+###############################################################################
+### Check for tests in the patch
+checkTests () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Checking there are new or changed tests in the patch."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ testReferences=`$GREP -c -i -e '^+++.*/test' $PATCH_DIR/patch`
+ echo "There appear to be $testReferences test files referenced in the patch."
+ if [[ $testReferences == 0 ]] ; then
+ if [[ $JENKINS == "true" ]] ; then
+ patchIsDoc=`$GREP -c -i 'title="documentation' $PATCH_DIR/jira`
+ if [[ $patchIsDoc != 0 ]] ; then
+ echo "The patch appears to be a documentation patch that doesn't require tests."
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+0 tests included{color}. The patch appears to be a documentation patch that doesn't require tests."
+ return 0
+ fi
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 tests included{color}. The patch doesn't appear to include any new or modified tests.
+ Please justify why no new tests are needed for this patch.
+ Also please list what manual steps were performed to verify this patch."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files."
+ return 0
+}
+
+cleanUpXml () {
+ if [ -d "${BASEDIR}/conf" ]
+ then
+ cd $BASEDIR/conf
+ for file in `ls *.xml.template`
+ do
+ if [ -f "${file}" ]
+ then
+ rm -f `basename ${file} .template`
+ fi
+ done
+ cd $BASEDIR
+ fi
+}
+
+###############################################################################
+### Attempt to apply the patch
+applyPatch () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Applying patch."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ export PATCH
+ $bindir/smart-apply-patch.sh $PATCH_DIR/patch
+ if [[ $? != 0 ]] ; then
+ echo "PATCH APPLICATION FAILED"
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 patch{color}. The patch command could not apply the patch."
+ return 1
+ fi
+ return 0
+}
+
+###############################################################################
+calculateJavadocWarnings() {
+ WARNING_FILE="$1"
+ RET=$(egrep "^[0-9]+ warnings?$" "$WARNING_FILE" | awk '{sum+=$1} END {print sum}')
+}
+
+### Check there are no javadoc warnings
+checkJavadocWarnings () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched javadoc warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "$MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1"
+ $MVN clean test javadoc:javadoc -DskipTests > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
+ calculateJavadocWarnings "$PATCH_DIR/masterJavadocWarnings.txt"
+ nummasterJavadocWarnings=$RET
+ calculateJavadocWarnings "$PATCH_DIR/patchJavadocWarnings.txt"
+ numPatchJavadocWarnings=$RET
+ grep -i warning "$PATCH_DIR/masterJavadocWarnings.txt" > "$PATCH_DIR/masterJavadocWarningsFiltered.txt"
+ grep -i warning "$PATCH_DIR/patchJavadocWarnings.txt" > "$PATCH_DIR/patchJavadocWarningsFiltered.txt"
+ diff -u "$PATCH_DIR/masterJavadocWarningsFiltered.txt" \
+ "$PATCH_DIR/patchJavadocWarningsFiltered.txt" > \
+ "$PATCH_DIR/diffJavadocWarnings.txt"
+ rm -f "$PATCH_DIR/masterJavadocWarningsFiltered.txt" "$PATCH_DIR/patchJavadocWarningsFiltered.txt"
+ echo "There appear to be $nummasterJavadocWarnings javadoc warnings before the patch and $numPatchJavadocWarnings javadoc warnings after applying the patch."
+ if [[ $nummasterJavadocWarnings != "" && $numPatchJavadocWarnings != "" ]] ; then
+ if [[ $numPatchJavadocWarnings -gt $nummasterJavadocWarnings ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 javadoc{color}. The javadoc tool appears to have generated `expr $(($numPatchJavadocWarnings-$nummasterJavadocWarnings))` warning messages.
+ See $BUILD_URL/artifact/patchprocess/diffJavadocWarnings.txt for details."
+ return 1
+ fi
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+1 javadoc{color}. There were no new javadoc warning messages."
+ return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Javac warnings
+checkJavacWarnings () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched javac warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "$MVN clean test -DskipTests > $PATCH_DIR/patchJavacWarnings.txt 2>&1"
+ $MVN clean test -DskipTests > $PATCH_DIR/patchJavacWarnings.txt 2>&1
+ if [[ $? != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 javac{color:red}. The patch appears to cause the build to fail."
+ return 2
+ fi
+ ### Compare master and patch javac warning numbers
+ if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
+ $GREP '\[WARNING\]' $PATCH_DIR/masterJavacWarnings.txt > $PATCH_DIR/filteredmasterJavacWarnings.txt
+ $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt > $PATCH_DIR/filteredPatchJavacWarnings.txt
+ masterJavacWarnings=`cat $PATCH_DIR/filteredmasterJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
+ patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
+ echo "There appear to be $masterJavacWarnings javac compiler warnings before the patch and $patchJavacWarnings javac compiler warnings after applying the patch."
+ if [[ $patchJavacWarnings != "" && $masterJavacWarnings != "" ]] ; then
+ if [[ $patchJavacWarnings -gt $masterJavacWarnings ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 javac{color}. The applied patch generated $patchJavacWarnings javac compiler warnings (more than the master's current $masterJavacWarnings warnings)."
+
+ $DIFF $PATCH_DIR/filteredmasterJavacWarnings.txt $PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt
+ JIRA_COMMENT_FOOTER="Javac warnings: $BUILD_URL/artifact/patchprocess/diffJavacWarnings.txt
+$JIRA_COMMENT_FOOTER"
+
+ return 1
+ fi
+ fi
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+1 javac{color}. The applied patch does not increase the total number of javac compiler warnings."
+ return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of release audit (RAT) warnings
+checkReleaseAuditWarnings () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched release audit warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "$MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1"
+ $MVN apache-rat:check > $PATCH_DIR/patchReleaseAuditOutput.txt 2>&1
+ find $BASEDIR -name rat.txt | xargs cat > $PATCH_DIR/patchReleaseAuditWarnings.txt
+
+ ### Compare master and patch release audit warning numbers
+ if [[ -f $PATCH_DIR/patchReleaseAuditWarnings.txt ]] ; then
+ patchReleaseAuditWarnings=`$GREP -c '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt`
+ echo ""
+ echo ""
+ echo "There appear to be $patchReleaseAuditWarnings release audit warnings after applying the patch."
+ if [[ $patchReleaseAuditWarnings != "" ]] ; then
+ if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 release audit{color}. The applied patch generated $patchReleaseAuditWarnings release audit warnings."
+ $GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt > $PATCH_DIR/patchReleaseAuditProblems.txt
+ echo "Lines that start with ????? in the release audit report indicate files that do not have an Apache license header." >> $PATCH_DIR/patchReleaseAuditProblems.txt
+ JIRA_COMMENT_FOOTER="Release audit warnings: $BUILD_URL/artifact/patchprocess/patchReleaseAuditProblems.txt
+$JIRA_COMMENT_FOOTER"
+ return 1
+ fi
+ fi
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+1 release audit{color}. The applied patch does not increase the total number of release audit warnings."
+ return 0
+}
+
+###############################################################################
+### Check there are no changes in the number of Checkstyle warnings
+checkStyle () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched checkstyle warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "$MVN test checkstyle:checkstyle -DskipTests > $PATCH_DIR/patchStyleErrors.txt 2>&1"
+ $MVN test checkstyle:checkstyle -DskipTests > $PATCH_DIR/patchStyleErrors.txt 2>&1
+
+# JIRA_COMMENT_FOOTER="Checkstyle results: $BUILD_URL/artifact/trunk/build/test/checkstyle-errors.html
+# $JIRA_COMMENT_FOOTER"
+
+ ### calculate actual patchStyleErrors
+ patchStyleErrors=0
+ if [[ -f target/munged/checkstyle-result.xml ]] ; then
+ $GREP -i '<error' target/munged/checkstyle-result.xml > $PATCH_DIR/filteredPatchCheckstyleWarnings.txt
+ patchStyleErrors=`cat $PATCH_DIR/filteredPatchCheckstyleWarnings.txt | $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
+ echo ""
+ echo ""
+ echo "There appear to be $patchStyleErrors checkstyle warnings after applying the patch."
+ fi
+
+ if [[ $patchStyleErrors != 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ -1 checkstyle. The patch generated $patchStyleErrors code style errors."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ +1 checkstyle. The patch generated 0 code style errors."
+ return 0
+}
+
+###############################################################################
+### Install the new jars so tests and findbugs can find all of the updated jars
+buildAndInstall () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Installing all of the jars"
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ echo "$MVN install -Dmaven.javadoc.skip=true -DskipTests -D${PROJECT_NAME}PatchProcess"
+ $MVN install -Dmaven.javadoc.skip=true -DskipTests
+ return $?
+}
+
+
+###############################################################################
+### Check there are no changes in the number of Findbugs warnings
+checkFindbugsWarnings () {
+ findbugs_version=`${FINDBUGS_HOME}/bin/findbugs -version`
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Determining number of patched Findbugs warnings."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+
+ rc=0
+ echo " Running findbugs "
+ echo "$MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1"
+ $MVN clean test findbugs:findbugs -DskipTests < /dev/null > $PATCH_DIR/patchFindBugsOutput.txt 2>&1
+ rc=$?
+
+ if [ $rc != 0 ] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 findbugs{color}. The patch appears to cause Findbugs (version ${findbugs_version}) to fail."
+ return 1
+ fi
+
+ findbugsWarnings=0
+ for file in $(find $BASEDIR -name findbugsXml.xml)
+ do
+ relative_file=${file#$BASEDIR/} # strip leading $BASEDIR prefix
+ if [ ! $relative_file == "target/findbugsXml.xml" ]; then
+ module_suffix=${relative_file%/target/findbugsXml.xml} # strip trailing path
+ module_suffix=`basename ${module_suffix}`
+ fi
+
+ cp $file $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
+ $FINDBUGS_HOME/bin/setBugDatabaseInfo -timestamp "01/01/2000" \
+ $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
+ $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml
+ newFindbugsWarnings=`$FINDBUGS_HOME/bin/filterBugs -first "01/01/2000" $PATCH_DIR/patchFindbugsWarnings${module_suffix}.xml \
+ $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml | $AWK '{print $1}'`
+ echo "Found $newFindbugsWarnings Findbugs warnings ($file)"
+ findbugsWarnings=$((findbugsWarnings+newFindbugsWarnings))
+ $FINDBUGS_HOME/bin/convertXmlToText -html \
+ $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.xml \
+ $PATCH_DIR/newPatchFindbugsWarnings${module_suffix}.html
+ if [[ $newFindbugsWarnings > 0 ]] ; then
+ JIRA_COMMENT_FOOTER="Findbugs warnings: $BUILD_URL/artifact/patchprocess/newPatchFindbugsWarnings${module_suffix}.html
+$JIRA_COMMENT_FOOTER"
+ fi
+ done
+
+ if [[ $findbugsWarnings -gt 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:red}-1 findbugs{color}. The patch appears to introduce $findbugsWarnings new Findbugs (version ${findbugs_version}) warnings."
+ return 1
+ fi
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+1 findbugs{color}. The patch does not introduce any new Findbugs (version ${findbugs_version}) warnings."
+ return 0
+}
+
+###############################################################################
+### Run the tests
+runTests () {
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Running tests."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+
+ failed_tests=""
+ failed_test_builds=""
+ test_timeouts=""
+ test_logfile=$PATCH_DIR/testrun.txt
+ echo " Running tests "
+ echo " $MVN clean install -fn -D${PROJECT_NAME}PatchProcess"
+ $MVN clean install -fae > $test_logfile 2>&1
+ test_build_result=$?
+ cat $test_logfile
+ module_test_timeouts=`$AWK '/^Running / { if (last) { print last } last=$2 } /^Tests run: / { last="" }' $test_logfile`
+ if [[ -n "$module_test_timeouts" ]] ; then
+ test_timeouts="$test_timeouts $module_test_timeouts"
+ fi
+ module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
+ if [[ -n "$module_failed_tests" ]] ; then
+ failed_tests="${failed_tests} ${module_failed_tests}"
+ fi
+ if [[ $test_build_result != 0 && -z "$module_failed_tests" && -z "$module_test_timeouts" ]] ; then
+ failed_test_builds="$module $failed_test_builds"
+ fi
+ result=0
+ comment_prefix=" {color:red}-1 core tests{color}."
+ if [[ -n "$failed_tests" ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+$comment_prefix The patch failed these unit tests in $modules:
+$failed_tests"
+ comment_prefix=" "
+ result=1
+ fi
+ if [[ -n "$test_timeouts" ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+$comment_prefix The following test timeouts occurred in $modules:
+$test_timeouts"
+ comment_prefix=" "
+ result=1
+ fi
+ if [[ -n "$failed_test_builds" ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+$comment_prefix The test build failed in $failed_test_builds"
+ result=1
+ fi
+ if [[ $result == 0 ]] ; then
+ JIRA_COMMENT="$JIRA_COMMENT
+
+ {color:green}+1 core tests{color}. The patch passed unit tests in $modules."
+ fi
+ return $result
+}
+
+###############################################################################
+# Find the maven module containing the given file.
+findModule (){
+ dir=`dirname $1`
+ while [ 1 ]
+ do
+ if [ -f "$dir/pom.xml" ]
+ then
+ echo $dir
+ return
+ else
+ dir=`dirname $dir`
+ fi
+ done
+}
+
+###############################################################################
+### Submit a comment to the defect's Jira
+submitJiraComment () {
+ local result=$1
+ ### Do not output the value of JIRA_COMMENT_FOOTER when run by a developer
+ if [[ $JENKINS == "false" ]] ; then
+ JIRA_COMMENT_FOOTER=""
+ fi
+ if [[ $result == 0 ]] ; then
+ comment="{color:green}+1 overall{color}. $JIRA_COMMENT
+
+$JIRA_COMMENT_FOOTER"
+ else
+ comment="{color:red}-1 overall{color}. $JIRA_COMMENT
+
+$JIRA_COMMENT_FOOTER"
+ fi
+ ### Output the test result to the console
+ echo "
+
+
+
+$comment"
+
+ if [[ $JENKINS == "true" ]] ; then
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Adding comment to Jira."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ ### Update Jira with a comment
+ export USER=hudson
+ $JIRACLI -s https://issues.apache.org/jira -a addcomment -u atlasqa -p $JIRA_PASSWD --comment "$comment" --issue $defect
+ $JIRACLI -s https://issues.apache.org/jira -a logout -u atlasqa -p $JIRA_PASSWD
+ fi
+}
+
+###############################################################################
+### Cleanup files
+cleanupAndExit () {
+ local result=$1
+ if [[ $JENKINS == "true" ]] ; then
+ if [ -e "$PATCH_DIR" ] ; then
+ mv $PATCH_DIR $BASEDIR
+ fi
+ fi
+ echo ""
+ echo ""
+ echo "======================================================================"
+ echo "======================================================================"
+ echo " Finished build."
+ echo "======================================================================"
+ echo "======================================================================"
+ echo ""
+ echo ""
+ exit $result
+}
+
+###############################################################################
+###############################################################################
+###############################################################################
+
+JIRA_COMMENT=""
+JIRA_COMMENT_FOOTER="Console output: $BUILD_URL/console
+
+This message is automatically generated."
+
+### Check if arguments to the script have been specified properly or not
+parseArgs $@
+cd $BASEDIR
+
+checkout
+RESULT=$?
+if [[ $JENKINS == "true" ]] ; then
+ if [[ $RESULT != 0 ]] ; then
+ exit 100
+ fi
+fi
+downloadPatch
+verifyPatch
+(( RESULT = RESULT + $? ))
+if [[ $RESULT != 0 ]] ; then
+ submitJiraComment 1
+ cleanupAndExit 1
+fi
+prebuildWithoutPatch
+(( RESULT = RESULT + $? ))
+if [[ $RESULT != 0 ]] ; then
+ submitJiraComment 1
+ cleanupAndExit 1
+fi
+checkAuthor
+(( RESULT = RESULT + $? ))
+
+if [[ $JENKINS == "true" ]] ; then
+ cleanUpXml
+fi
+checkTests
+(( RESULT = RESULT + $? ))
+applyPatch
+APPLY_PATCH_RET=$?
+(( RESULT = RESULT + $APPLY_PATCH_RET ))
+if [[ $APPLY_PATCH_RET != 0 ]] ; then
+ submitJiraComment 1
+ cleanupAndExit 1
+fi
+checkJavacWarnings
+JAVAC_RET=$?
+#2 is returned if the code could not compile
+if [[ $JAVAC_RET == 2 ]] ; then
+ submitJiraComment 1
+ cleanupAndExit 1
+fi
+(( RESULT = RESULT + $JAVAC_RET ))
+checkJavadocWarnings
+(( RESULT = RESULT + $? ))
+checkStyle
+(( RESULT = RESULT + $? ))
+checkFindbugsWarnings
+(( RESULT = RESULT + $? ))
+checkReleaseAuditWarnings
+(( RESULT = RESULT + $? ))
+buildAndInstall
+### Run tests for Jenkins or if explictly asked for by a developer
+if [[ $JENKINS == "true" || $RUN_TESTS == "true" ]] ; then
+ runTests
+ (( RESULT = RESULT + $? ))
+fi
+JIRA_COMMENT_FOOTER="Test results: $BUILD_URL/testReport/
+$JIRA_COMMENT_FOOTER"
+
+submitJiraComment $RESULT
+cleanupAndExit $RESULT
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bd47628f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 095a8dd..c74e31f 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1583,6 +1583,7 @@
<exclude>.bowerrc</exclude>
<exclude>*.json</exclude>
<exclude>**/overlays/**</exclude>
+ <exclude>dev-support/**</exclude>
</excludes>
</configuration>
<executions>
@@ -1697,4 +1698,4 @@
</plugin>
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>