You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/04/21 22:23:23 UTC
[3/3] hadoop git commit: HADOOP-11746. rewrite test-patch.sh (aw)
HADOOP-11746. rewrite test-patch.sh (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7251723
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7251723
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7251723
Branch: refs/heads/branch-2
Commit: b7251723d8347b764b0792e1b6bc4d9a03b71338
Parents: 447f2f6
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Apr 21 21:22:09 2015 +0100
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Apr 21 21:22:09 2015 +0100
----------------------------------------------------------------------
dev-support/test-patch.d/checkstyle.sh | 149 +
dev-support/test-patch.d/shellcheck.sh | 138 +
dev-support/test-patch.d/whitespace.sh | 40 +
dev-support/test-patch.sh | 2856 +++++++++++++-----
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
5 files changed, 2491 insertions(+), 694 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7251723/dev-support/test-patch.d/checkstyle.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh
new file mode 100755
index 0000000..460709e
--- /dev/null
+++ b/dev-support/test-patch.d/checkstyle.sh
@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin checkstyle
+
+CHECKSTYLE_TIMER=0
+
+# if it ends in an explicit .sh, then this is shell code.
+# if it doesn't have an extension, we assume it is shell code too
+function checkstyle_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test checkstyle
+ fi
+}
+
+function checkstyle_preapply
+{
+ verify_needed_test checkstyle
+
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "checkstyle plugin: prepatch"
+
+ start_clock
+ echo_and_redirect "${PATCH_DIR}/${PATCH_BRANCH}checkstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
+ if [[ $? != 0 ]] ; then
+ echo "Pre-patch ${PATCH_BRANCH} checkstyle compilation is broken?"
+ add_jira_table -1 checkstyle "Pre-patch ${PATCH_BRANCH} checkstyle compilation may be broken."
+ return 1
+ fi
+
+ cp -p "${BASEDIR}/target/checkstyle-result.xml" \
+ "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml"
+
+ # keep track of how much as elapsed for us already
+ CHECKSTYLE_TIMER=$(stop_clock)
+ return 0
+}
+
+function checkstyle_postapply
+{
+ verify_needed_test checkstyle
+
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "checkstyle plugin: postpatch"
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${CHECKSTYLE_TIMER}"
+
+ echo_and_redirect "${PATCH_DIR}/patchcheckstyle.txt" "${MVN}" test checkstyle:checkstyle-aggregate -DskipTests "-D${PROJECT_NAME}PatchProcess"
+ if [[ $? != 0 ]] ; then
+ echo "Post-patch checkstyle compilation is broken."
+ add_jira_table -1 checkstyle "Post-patch checkstyle compilation is broken."
+ return 1
+ fi
+
+ cp -p "${BASEDIR}/target/checkstyle-result.xml" \
+ "${PATCH_DIR}/checkstyle-result-patch.xml"
+
+ checkstyle_runcomparison
+
+ # shellcheck disable=SC2016
+ CHECKSTYLE_POSTPATCH=$(wc -l "${PATCH_DIR}/checkstyle-result-diff.txt" | ${AWK} '{print $1}')
+
+ if [[ ${CHECKSTYLE_POSTPATCH} -gt 0 ]] ; then
+
+ add_jira_table -1 checkstyle "The applied patch generated "\
+ "${CHECKSTYLE_POSTPATCH}" \
+ " additional checkstyle issues."
+ add_jira_footer checkstyle "@@BASE@@/checkstyle-result-diff.txt"
+
+ return 1
+ fi
+ add_jira_table +1 checkstyle "There were no new checkstyle issues."
+ return 0
+}
+
+
+function checkstyle_runcomparison
+{
+
+ python <(cat <<EOF
+import os
+import sys
+import xml.etree.ElementTree as etree
+from collections import defaultdict
+
+if len(sys.argv) != 3 :
+ print "usage: %s checkstyle-result-master.xml checkstyle-result-patch.xml" % sys.argv[0]
+ exit(1)
+
+def path_key(x):
+ path = x.attrib['name']
+ return path[path.find('${PROJECT_NAME}-'):]
+
+def print_row(path, master_errors, patch_errors):
+ print '%s\t%s\t%s' % (k,master_dict[k],child_errors)
+
+master = etree.parse(sys.argv[1])
+patch = etree.parse(sys.argv[2])
+
+master_dict = defaultdict(int)
+
+for child in master.getroot().getchildren():
+ if child.tag != 'file':
+ continue
+ child_errors = len(child.getchildren())
+ if child_errors == 0:
+ continue
+ master_dict[path_key(child)] = child_errors
+
+for child in patch.getroot().getchildren():
+ if child.tag != 'file':
+ continue
+ child_errors = len(child.getchildren())
+ if child_errors == 0:
+ continue
+ k = path_key(child)
+ if child_errors > master_dict[k]:
+ print_row(k, master_dict[k], child_errors)
+
+EOF
+) "${PATCH_DIR}/checkstyle-result-${PATCH_BRANCH}.xml" "${PATCH_DIR}/checkstyle-result-patch.xml" > "${PATCH_DIR}/checkstyle-result-diff.txt"
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7251723/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
new file mode 100755
index 0000000..b60da36
--- /dev/null
+++ b/dev-support/test-patch.d/shellcheck.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin shellcheck
+
+SHELLCHECK_TIMER=0
+
+SHELLCHECK=${SHELLCHECK:-$(which shellcheck)}
+
+SHELLCHECK_SPECIFICFILES=""
+
+# if it ends in an explicit .sh, then this is shell code.
+# if it doesn't have an extension, we assume it is shell code too
+function shellcheck_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.sh$ ]]; then
+ add_test shellcheck
+ SHELLCHECK_SPECIFICFILES="${SHELLCHECK_SPECIFICFILES} ./${filename}"
+ fi
+
+ if [[ ! ${filename} =~ \. ]]; then
+ add_test shellcheck
+ fi
+}
+
+function shellcheck_private_findbash
+{
+ local i
+ local value
+ local list
+
+ while read line; do
+ value=$(find "${line}" ! -name '*.cmd' -type f \
+ | ${GREP} -E -v '(.orig$|.rej$)')
+ list="${list} ${value}"
+ done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
+ # shellcheck disable=SC2086
+ echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
+}
+
+function shellcheck_preapply
+{
+ local i
+
+ verify_needed_test shellcheck
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "shellcheck plugin: prepatch"
+
+ if [[ -z "${SHELLCHECK}" ]]; then
+ hadoop_error "shellcheck is not available."
+ fi
+
+ start_clock
+
+ # shellcheck disable=SC2016
+ SHELLCHECK_VERSION=$(shellcheck --version | ${GREP} version: | ${AWK} '{print $NF}')
+
+ echo "Running shellcheck against all identifiable shell scripts"
+ pushd "${BASEDIR}" >/dev/null
+ for i in $(shellcheck_private_findbash); do
+ if [[ -f ${i} ]]; then
+ ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt"
+ fi
+ done
+ popd > /dev/null
+ # keep track of how much as elapsed for us already
+ SHELLCHECK_TIMER=$(stop_clock)
+ return 0
+}
+
+function shellcheck_postapply
+{
+ local i
+
+ verify_needed_test shellcheck
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "shellcheck plugin: postpatch"
+
+ if [[ -z "${SHELLCHECK}" ]]; then
+ hadoop_error "shellcheck is not available."
+ fi
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${SHELLCHECK_TIMER}"
+
+ echo "Running shellcheck against all identifiable shell scripts"
+ # we re-check this in case one has been added
+ for i in $(shellcheck_private_findbash); do
+ ${SHELLCHECK} -f gcc "${i}" >> "${PATCH_DIR}/patchshellcheck-result.txt"
+ done
+
+ # shellcheck disable=SC2016
+ numPrepatch=$(wc -l "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" | ${AWK} '{print $1}')
+ # shellcheck disable=SC2016
+ numPostpatch=$(wc -l "${PATCH_DIR}/patchshellcheck-result.txt" | ${AWK} '{print $1}')
+
+ ${DIFF} -u "${PATCH_DIR}/${PATCH_BRANCH}shellcheck-result.txt" \
+ "${PATCH_DIR}/patchshellcheck-result.txt" \
+ | ${GREP} '^+\.' \
+ > "${PATCH_DIR}/diffpatchshellcheck.txt"
+
+ # shellcheck disable=SC2016
+ diffPostpatch=$(wc -l "${PATCH_DIR}/diffpatchshellcheck.txt" | ${AWK} '{print $1}')
+
+ if [[ ${diffPostpatch} -gt 0 ]] ; then
+ add_jira_table -1 shellcheck "The applied patch generated "\
+ "${diffPostpatch} new shellcheck (v${SHELLCHECK_VERSION}) issues (total was ${numPrepatch}, now ${numPostpatch})."
+ add_jira_footer shellcheck "@@BASE@@/diffpatchshellcheck.txt"
+ return 1
+ fi
+
+ add_jira_table +1 shellcheck "There were no new shellcheck (v${SHELLCHECK_VERSION}) issues."
+ return 0
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7251723/dev-support/test-patch.d/whitespace.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh
new file mode 100755
index 0000000..deac654
--- /dev/null
+++ b/dev-support/test-patch.d/whitespace.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin whitespace
+
+function whitespace_preapply
+{
+ local count
+
+ big_console_header "Checking for whitespace at the end of lines"
+ start_clock
+
+ ${GREP} '^+' "${PATCH_DIR}/patch" | ${GREP} '[[:blank:]]$' > "${PATCH_DIR}/whitespace.txt"
+
+ # shellcheck disable=SC2016
+ count=$(wc -l "${PATCH_DIR}/whitespace.txt" | ${AWK} '{print $1}')
+
+ if [[ ${count} -gt 0 ]]; then
+ add_jira_table -1 whitespace "The patch has ${count}"\
+ " line(s) that end in whitespace."
+ add_jira_footer whitespace "@@BASE@@/whitespace.txt"
+ return 1
+ fi
+
+ add_jira_table +1 whitespace "The patch has no lines that end in whitespace."
+ return 0
+}