You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by bu...@apache.org on 2015/09/23 04:23:31 UTC
[01/50] [abbrv] yetus git commit: HADOOP-12298. Move recent yetus
docker changes to the yetus dockerfile (aw)
Repository: yetus
Updated Branches:
refs/heads/master 8cf07f20f -> 2db6e7a1a
HADOOP-12298. Move recent yetus docker changes to the yetus dockerfile (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/91e48da4
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/91e48da4
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/91e48da4
Branch: refs/heads/master
Commit: 91e48da4a7ecbea651008ef15df370d3d66fb7c2
Parents: 5af528a
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jul 30 14:42:27 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jul 30 14:42:27 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch-docker/Dockerfile-startstub | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/91e48da4/dev-support/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/Dockerfile-startstub b/dev-support/test-patch-docker/Dockerfile-startstub
index e534f14..5e5ca78 100644
--- a/dev-support/test-patch-docker/Dockerfile-startstub
+++ b/dev-support/test-patch-docker/Dockerfile-startstub
@@ -32,7 +32,8 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
libjansson-dev \
fuse libfuse-dev \
libcurl4-openssl-dev \
- python python2.7 \
+ python python2.7 pylint \
+ ruby \
openjdk-7-jdk
# Fixing the Apache commons / Maven dependency problem under Ubuntu:
@@ -71,6 +72,11 @@ ENV FINDBUGS_HOME /opt/findbugs
RUN apt-get install -y cabal-install
RUN cabal update && cabal install shellcheck --global
+####
+# Install rubocop
+###
+RUN gem install rubocop
+
#####
# Install JIRA CLI
#####
[03/50] [abbrv] yetus git commit: HADOOP-12255. Add support for
rubocop (missed a file) (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12255. Add support for rubocop (missed a file) (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/10a5f807
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/10a5f807
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/10a5f807
Branch: refs/heads/master
Commit: 10a5f807906ac7e0cec2b61cf1aed97a6d3bf614
Parents: 39aebf6
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jul 30 17:06:42 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jul 30 17:06:42 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/rubocop.sh | 140 +++++++++++++++++++++++++++++++
1 file changed, 140 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/10a5f807/dev-support/test-patch.d/rubocop.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/rubocop.sh b/dev-support/test-patch.d/rubocop.sh
new file mode 100755
index 0000000..ba9810e
--- /dev/null
+++ b/dev-support/test-patch.d/rubocop.sh
@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin rubocop
+
+RUBOCOP_TIMER=0
+
+RUBOCOP=${RUBOCOP:-$(which rubocop 2>/dev/null)}
+
+function rubocop_usage
+{
+ echo "Rubocop specific:"
+ echo "--rubocop=<path> path to rubocop executable"
+}
+
+function rubocop_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --rubocop=*)
+ RUBOCOP=${i#*=}
+ ;;
+ esac
+ done
+}
+
+function rubocop_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.rb$ ]]; then
+ add_test rubocop
+ fi
+}
+
+function rubocop_preapply
+{
+ local i
+
+ verify_needed_test rubocop
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "rubocop plugin: prepatch"
+
+ if [[ ! -x ${RUBOCOP} ]]; then
+ yetus_error "${RUBOCOP} does not exist."
+ return 0
+ fi
+
+ start_clock
+
+ echo "Running rubocop against modified ruby scripts."
+ pushd "${BASEDIR}" >/dev/null
+ for i in ${CHANGED_FILES}; do
+ if [[ ${i} =~ \.rb$ && -f ${i} ]]; then
+ ${RUBOCOP} -f c "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/branch-rubocop-result.txt"
+ fi
+ done
+ popd >/dev/null
+ # keep track of how much as elapsed for us already
+ RUBOCOP_TIMER=$(stop_clock)
+ return 0
+}
+
+function rubocop_postapply
+{
+ local i
+ local numPrepatch
+ local numPostpatch
+ local diffPostpatch
+
+ verify_needed_test rubocop
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "rubocop plugin: postpatch"
+
+ if [[ ! -x ${RUBOCOP} ]]; then
+ yetus_error "${RUBOCOP} is not available."
+ add_vote_table 0 rubocop "Rubocop was not available."
+ return 0
+ fi
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${RUBOCOP_TIMER}"
+
+ echo "Running rubocop against modified ruby scripts."
+ # we re-check this in case one has been added
+ pushd "${BASEDIR}" >/dev/null
+ for i in ${CHANGED_FILES}; do
+ if [[ ${i} =~ \.rb$ && -f ${i} ]]; then
+ ${RUBOCOP} -f c "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/patch-rubocop-result.txt"
+ fi
+ done
+ popd >/dev/null
+
+ # shellcheck disable=SC2016
+ RUBOCOP_VERSION=$(${RUBOCOP} -v | ${AWK} '{print $NF}')
+ add_footer_table rubocop "v${RUBOCOP_VERSION}"
+
+ calcdiffs "${PATCH_DIR}/branch-rubocop-result.txt" "${PATCH_DIR}/patch-rubocop-result.txt" > "${PATCH_DIR}/diff-patch-rubocop.txt"
+ diffPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-rubocop.txt")
+
+ if [[ ${diffPostpatch} -gt 0 ]] ; then
+ # shellcheck disable=SC2016
+ numPrepatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-rubocop-result.txt")
+
+ # shellcheck disable=SC2016
+ numPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-rubocop-result.txt")
+
+ add_vote_table -1 rubocop "The applied patch generated "\
+ "${diffPostpatch} new rubocop issues (total was ${numPrepatch}, now ${numPostpatch})."
+ add_footer_table rubocop "@@BASE@@/diff-patch-rubocop.txt"
+ return 1
+ fi
+
+ add_vote_table +1 rubocop "There were no new rubocop issues."
+ return 0
+}
[05/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/2cebff7f
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/2cebff7f
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/2cebff7f
Branch: refs/heads/master
Commit: 2cebff7f4dfb4b65cf03fdb82a625f2aa90f2bab
Parents: 066e118 c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jul 30 18:41:29 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jul 30 18:41:29 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[04/50] [abbrv] yetus git commit: HADOOP-12228. Document
releasedocmaker (fix the commit) (aw)
Posted by bu...@apache.org.
HADOOP-12228. Document releasedocmaker (fix the commit) (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/066e1180
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/066e1180
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/066e1180
Branch: refs/heads/master
Commit: 066e11806878c2f02ce22a84515a73335da91e7c
Parents: 10a5f80
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jul 30 17:10:33 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jul 30 17:10:33 2015 -0700
----------------------------------------------------------------------
dev-support/docs/releasedocmaker.md | 115 -------------------------------
1 file changed, 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/066e1180/dev-support/docs/releasedocmaker.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/releasedocmaker.md b/dev-support/docs/releasedocmaker.md
index d8eb13c..b39de2e 100644
--- a/dev-support/docs/releasedocmaker.md
+++ b/dev-support/docs/releasedocmaker.md
@@ -113,118 +113,3 @@ $ releasedocmaker.py --project HBASE --version 1.0.0 --lint
```
This will do the normal JIRA querying, looking for items it considers problematic. It will print the information to the screen and then exit with either success or failure, depending upon if any issues were discovered.
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-releasedocmaker
-===============
-
-* [Purpose](#Purpose)
-* [Basic Usage](#Basic_Usage)
-* [Changing the Header](#Changing_the_Header)
-* [Multiple Versions](#Multiple_Versions)
-* [Unreleased Dates](#Unreleased_Dates)
-* [Lint Mode](#Lint_Mode)
-
-# Purpose
-
-Building changelog information in a form that is human digestible but still containing as much useful information is difficult. Many attempts over the years have resulted in a variety of methods that projects use to solve this problem:
-
-* JIRA-generated release notes from the "Release Notes" button
-* Manually modified CHANGES file
-* Processing git log information
-
-All of these methods have their pros and cons. Some have issues with accuracy. Some have issues with lack of details. None of these methods seem to cover all of the needs of many projects and are full of potential pitfalls.
-
-In order to solve these problems, releasedocmaker was written to automatically generate a changelog and release notes by querying Apache's JIRA instance.
-
-# Basic Usage
-
-Minimally, the name of the JIRA project and a version registered in JIRA must be provided:
-
-```bash
-$ releasedocmaker.py --project (project) --version (version)
-```
-
-This will query Apache JIRA, generating two files in a directory named after the given version in an extended markdown format which can be processed by both mvn site and GitHub.
-
-* CHANGES.(version).md
-
-This is similar to the JIRA "Release Notes" button but is in tabular format and includes the priority, component, reporter, and contributor fields. It also highlights Incompatible Changes so that readers know what to look out for when upgrading. The top of the file also includes the date that the version was marked as released in JIRA.
-
-
-* RELEASENOTES.(version).md
-
-If your JIRA project supports the release note field, this will contain any JIRA mentioned in the CHANGES log that is either an incompatible change or has a release note associated with it. If your JIRA project does not support the release notes field, this will be the description field.
-
-For example, to build the release documentation for HBase v1.2.0...
-
-```bash
-$ releasedocmaker.py --project HBASE --version 1.2.0
-```
-
-... will create a 1.2.0 directory and inside that directory will be CHANGES.1.2.0.md and RELEASENOTES.1.2.0.md .
-
-
-# Changing the Header
-
-By default, it will use a header that matches the project name. But that is kind of ugly and the case may be wrong. Luckily, the title can be changed:
-
-```bash
-$ releasedocmaker.py --project HBASE --version 1.2.0 --projecttitle "Apache HBase"
-```
-
-Now instead of "HBASE", it will use "Apache HBASE" for some titles and headers.
-
-# Multiple Versions
-
-The script can also generate multiple versions at once, by
-
-```bash
-$ releasedocmaker.py --project HBASE --version 1.0.0 --version 1.2.0
-```
-
-This will create the files for versions 1.0.0 and versions 1.2.0 in their own directories.
-
-But what if the version numbers are not known? releasedocmaker can also generate version data based upon ranges:
-
-```bash
-$ releasedocmaker.py --project HBASE --version 1.0.0 --version 1.2.0 --range
-```
-
-In this form, releasedocmaker will query JIRA, discover all versions that alphabetically appear to be between 1.0.0 and 1.2.0, inclusive, and generate all of the relative release documents. This is especially useful when bootstrapping an existing project.
-
-# Unreleased Dates
-
-For released versions, releasedocmaker will pull the date of the release from JIRA. However, for unreleased versions it marks the release as "Unreleased". This can be inconvenient when actually building a release and wanting to include it inside the source package.
-
-The --usetoday option can be used to signify that instead of using Unreleased, releasedocmaker should use today's date.
-
-```bash
-$ releasedocmaker.py --project HBASE --version 1.0.0 --usetoday
-```
-
-After using this option and release, don't forget to change JIRA's release date to match!
-
-# Lint Mode
-
-In order to ensure proper formatting while using mvn site, releasedocmaker puts in periods (.) for fields that are empty or unassigned. This can be unsightly and not proper for any given project. There are also other things, such as missing release notes for incompatible changes, that are less than desirable.
-
-In order to help release managers from having to scan through potentially large documents, releasedocmaker features a lint mode, triggered via --lint:
-
-```bash
-$ releasedocmaker.py --project HBASE --version 1.0.0 --lint
-```
-
-This will do the normal JIRA querying, looking for items it considers problematic. It will print the information to the screen and then exit with either success or failure, depending upon if any issues were discovered.
[42/50] [abbrv] yetus git commit: HADOOP-12375. Incomplete checking
for findbugs executable (Jagadesh Kiran N via aw)
Posted by bu...@apache.org.
HADOOP-12375. Incomplete checking for findbugs executable (Jagadesh Kiran N via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/677bfe03
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/677bfe03
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/677bfe03
Branch: refs/heads/master
Commit: 677bfe0330c95904be0c6bec1c0b7b3183cfd1e7
Parents: c702233
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 4 14:47:18 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 4 14:47:18 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/findbugs.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/677bfe03/dev-support/test-patch.d/findbugs.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/findbugs.sh b/dev-support/test-patch.d/findbugs.sh
index 16ef8d7..df479af 100755
--- a/dev-support/test-patch.d/findbugs.sh
+++ b/dev-support/test-patch.d/findbugs.sh
@@ -64,7 +64,7 @@ function findbugs_parse_args
## @return 1 findbugs is missing some component
function findbugs_is_installed
{
- if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
+ if [[ ! -x "${FINDBUGS_HOME}/bin/findbugs" ]]; then
printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
add_vote_table -1 findbugs "Findbugs is not installed."
return 1
[46/50] [abbrv] yetus git commit: HADOOP-12257. rework build tool
support; add gradle; add scala (aw)
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/scala.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/scala.sh b/dev-support/test-patch.d/scala.sh
new file mode 100755
index 0000000..4c88db4
--- /dev/null
+++ b/dev-support/test-patch.d/scala.sh
@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin scalac
+add_plugin scaladoc
+
+function scalac_filefilter
+{
+ declare filename=$1
+
+ if [[ ${filename} =~ \.scala$ ]]; then
+ yetus_debug "tests/scalac: ${filename}"
+ add_test scalac
+ add_test compile
+ fi
+}
+
+function scaladoc_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.scala$ ]]; then
+ yetus_debug "tests/scaladoc: ${filename}"
+ add_test scaladoc
+ fi
+}
+
+## @description
+## @audience private
+## @stability stable
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function scalac_compile
+{
+ declare codebase=$1
+ declare multijdkmode=$2
+
+ verify_needed_test scalac
+ if [[ $? = 0 ]]; then
+ return 0
+ fi
+
+ if [[ ${codebase} = patch ]]; then
+ generic_postlog_compare compile scalac "${multijdkmode}"
+ fi
+}
+
+## @description Count and compare the number of JavaDoc warnings pre- and post- patch
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function scaladoc_rebuild
+{
+ declare codebase=$1
+
+ if [[ "${codebase}" = branch ]]; then
+ generic_pre_handler scaladoc false
+ else
+ generic_post_handler scaladoc scaladoc false true
+ fi
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
index 0c198db..3cc7f6e 100755
--- a/dev-support/test-patch.d/shellcheck.sh
+++ b/dev-support/test-patch.d/shellcheck.sh
@@ -161,3 +161,14 @@ function shellcheck_postapply
add_vote_table +1 shellcheck "There were no new shellcheck issues."
return 0
}
+
+function shellcheck_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ shellcheck_preapply
+ else
+ shellcheck_postapply
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/test4tests.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/test4tests.sh b/dev-support/test-patch.d/test4tests.sh
new file mode 100755
index 0000000..bd214a2
--- /dev/null
+++ b/dev-support/test-patch.d/test4tests.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin test4tests
+
+## @description Check the patch file for changed/new tests
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function test4tests_patchfile
+{
+ declare testReferences=0
+ declare i
+
+ big_console_header "Checking there are new or changed tests in the patch."
+
+ verify_needed_test unit
+
+ if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need new or modified tests."
+ return 0
+ fi
+
+ start_clock
+
+ for i in ${CHANGED_FILES}; do
+ if [[ ${i} =~ (^|/)test/ ]]; then
+ ((testReferences=testReferences + 1))
+ fi
+ done
+
+ echo "There appear to be ${testReferences} test file(s) referenced in the patch."
+ if [[ ${testReferences} == 0 ]] ; then
+ add_vote_table -1 "test4tests" \
+ "The patch doesn't appear to include any new or modified tests. " \
+ "Please justify why no new tests are needed for this patch." \
+ "Also please list what manual steps were performed to verify this patch."
+ return 1
+ fi
+ add_vote_table +1 "test4tests" \
+ "The patch appears to include ${testReferences} new or modified test files."
+ return 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/whitespace.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh
index bab32dd..b16080f 100755
--- a/dev-support/test-patch.d/whitespace.sh
+++ b/dev-support/test-patch.d/whitespace.sh
@@ -16,17 +16,16 @@
add_plugin whitespace
-
function whitespace_linecomment_reporter
{
- local file=$1
+ declare file=$1
shift
- local comment=$*
- local tmpfile="${PATCH_DIR}/wlr.$$.${RANDOM}"
+ declare comment=$*
+ declare tmpfile="${PATCH_DIR}/wlr.$$.${RANDOM}"
while read -r line; do
{
- #shellcheck disable=SC2086
+ # shellcheck disable=SC2086
printf "%s" "$(echo ${line} | cut -f1-2 -d:)"
echo "${comment}"
} >> "${tmpfile}"
@@ -36,10 +35,15 @@ function whitespace_linecomment_reporter
rm "${tmpfile}"
}
-function whitespace_postapply
+function whitespace_postcompile
{
- local count
- local result=0
+ declare repostatus=$1
+ declare count
+ declare result=0
+
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
big_console_header "Checking for whitespace at the end of lines"
start_clock
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/xml.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/xml.sh b/dev-support/test-patch.d/xml.sh
index b33c7cd..b59df5c 100755
--- a/dev-support/test-patch.d/xml.sh
+++ b/dev-support/test-patch.d/xml.sh
@@ -18,24 +18,29 @@ add_plugin xml
function xml_filefilter
{
- local filename=$1
+ declare filename=$1
if [[ ${filename} =~ \.xml$ ]]; then
add_test xml
fi
}
-function xml_postapply
+function xml_postcompile
{
- local js
- local i
- local count
+ declare repostatus=$1
+ declare js
+ declare i
+ declare count
verify_needed_test xml
if [[ $? == 0 ]]; then
return 0
fi
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
+
big_console_header "Checking if XML files are well-formed"
js="${JAVA_HOME}/bin/jrunscript"
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 4d4b63f..f311076 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -34,8 +34,6 @@ GLOBALTIMER=$(date +"%s")
QATESTMODE=false
# global arrays
-declare -a MAVEN_ARGS=("--batch-mode")
-declare -a ANT_ARGS=("-noinput")
declare -a TP_HEADER
declare -a TP_VOTE_TABLE
declare -a TP_TEST_TABLE
@@ -44,6 +42,7 @@ declare -a MODULE_STATUS
declare -a MODULE_STATUS_TIMER
declare -a MODULE_STATUS_MSG
declare -a MODULE_STATUS_LOG
+declare -a MODULE_COMPILE_LOG
declare -a MODULE
TP_HEADER_COUNTER=0
@@ -57,18 +56,6 @@ TP_FOOTER_COUNTER=0
## @replaceable no
function setup_defaults
{
- if [[ -z "${MAVEN_HOME:-}" ]]; then
- MVN=mvn
- else
- MVN=${MAVEN_HOME}/bin/mvn
- fi
-
- if [[ -z "${ANT_HOME:-}" ]]; then
- ANT=ant
- else
- ANT=${ANT_HOME}/bin/ant
- fi
-
PROJECT_NAME=yetus
DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub"
HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
@@ -81,10 +68,10 @@ function setup_defaults
ALLOWSUMMARIES=true
DOCKERSUPPORT=false
- ECLIPSE_HOME=${ECLIPSE_HOME:-}
BUILD_NATIVE=${BUILD_NATIVE:-true}
PATCH_BRANCH=""
PATCH_BRANCH_DEFAULT="master"
+ BUILDTOOLCWD=true
# shellcheck disable=SC2034
CHANGED_MODULES=""
@@ -102,7 +89,7 @@ function setup_defaults
OSTYPE=$(uname -s)
BUILDTOOL=maven
TESTFORMATS=""
- JDK_TEST_LIST="javac javadoc unit"
+ JDK_TEST_LIST="compile javadoc unit"
# Solaris needs POSIX, not SVID
case ${OSTYPE} in
@@ -227,7 +214,33 @@ function stop_global_clock
## @param seconds
function offset_clock
{
- ((TIMER=TIMER-$1))
+ declare off=$1
+
+ yetus_debug "offset clock by ${off}"
+
+ if [[ -n ${off} ]]; then
+ ((TIMER=TIMER-off))
+ else
+ yetus_error "ASSERT: no offset passed to offset_clock: ${index}"
+ generate_stack
+ fi
+}
+
+## @description generate a stack trace when in debug mode
+## @audience public
+## @stability stable
+## @replaceable no
+## @return exits
+function generate_stack
+{
+ declare frame
+
+ if [[ -n "${TP_SHELL_SCRIPT_DEBUG}" ]]; then
+ while caller "${frame}"; do
+ ((frame++));
+ done
+ fi
+ exit 1
}
## @description Add to the header of the display
@@ -319,30 +332,6 @@ function verify_multijdk_test
return 0
}
-## @description Absolute path the JDK_DIR_LIST and JAVA_HOME.
-## @description if JAVA_HOME is in JDK_DIR_LIST, it is positioned last
-## @stability stable
-## @audience private
-## @replaceable yes
-function fullyqualifyjdks
-{
- local i
- local jdkdir
- local tmplist
-
- JAVA_HOME=$(cd -P -- "${JAVA_HOME}" >/dev/null && pwd -P)
-
- for i in ${JDK_DIR_LIST}; do
- jdkdir=$(cd -P -- "${i}" >/dev/null && pwd -P)
- if [[ ${jdkdir} != "${JAVA_HOME}" ]]; then
- tmplist="${tmplist} ${jdkdir}"
- fi
- done
-
- JDK_DIR_LIST="${tmplist} ${JAVA_HOME}"
- JDK_DIR_LIST=${JDK_DIR_LIST/ }
-}
-
## @description Put the opening environment information at the bottom
## @description of the footer table
## @stability stable
@@ -350,11 +339,8 @@ function fullyqualifyjdks
## @replaceable yes
function prepopulate_footer
{
- # shellcheck disable=SC2016
- local javaversion
- local listofjdks
- local -r unamea=$(uname -a)
- local i
+ # shellcheck disable=SC2155
+ declare -r unamea=$(uname -a)
add_footer_table "uname" "${unamea}"
add_footer_table "Build tool" "${BUILDTOOL}"
@@ -362,17 +348,6 @@ function prepopulate_footer
if [[ -n ${PERSONALITY} ]]; then
add_footer_table "Personality" "${PERSONALITY}"
fi
-
- javaversion=$(report_jvm_version "${JAVA_HOME}")
- add_footer_table "Default Java" "${javaversion}"
- if [[ -n ${JDK_DIR_LIST}
- && ${JDK_DIR_LIST} != "${JAVA_HOME}" ]]; then
- for i in ${JDK_DIR_LIST}; do
- javaversion=$(report_jvm_version "${i}")
- listofjdks="${listofjdks} ${i}:${javaversion}"
- done
- add_footer_table "Multi-JDK versions" "${listofjdks}"
- fi
}
## @description Put docker stats in various tables
@@ -505,40 +480,6 @@ function findlargest
echo "${maxlen}"
}
-## @description Verify that ${JAVA_HOME} is defined
-## @audience public
-## @stability stable
-## @replaceable no
-## @return 1 - no JAVA_HOME
-## @return 0 - JAVA_HOME defined
-function find_java_home
-{
- start_clock
- if [[ -z ${JAVA_HOME:-} ]]; then
- case ${OSTYPE} in
- Darwin)
- if [[ -z "${JAVA_HOME}" ]]; then
- if [[ -x /usr/libexec/java_home ]]; then
- JAVA_HOME="$(/usr/libexec/java_home)"
- export JAVA_HOME
- else
- export JAVA_HOME=/Library/Java/Home
- fi
- fi
- ;;
- *)
- ;;
- esac
- fi
-
- if [[ -z ${JAVA_HOME:-} ]]; then
- echo "JAVA_HOME is not defined."
- add_vote_table -1 pre-patch "JAVA_HOME is not defined."
- return 1
- fi
- return 0
-}
-
## @description Write the contents of a file to all of the bug systems
## @description (so content should avoid special formatting)
## @params filename
@@ -691,13 +632,15 @@ function compute_unidiff
for fn in ${CHANGED_FILES}; do
filen=${fn##./}
- ${GIT} diff ${filen} \
- | tail -n +6 \
- | ${GREP} -n '^+' \
- | ${GREP} -vE '^[0-9]*:\+\+\+' \
- | ${SED} -e 's,^\([0-9]*:\)\+,\1,g' \
- -e s,^,./${filen}:,g \
- >> "${tmpfile}"
+ if [[ -f "${filen}" ]]; then
+ ${GIT} diff ${filen} \
+ | tail -n +6 \
+ | ${GREP} -n '^+' \
+ | ${GREP} -vE '^[0-9]*:\+\+\+' \
+ | ${SED} -e 's,^\([0-9]*:\)\+,\1,g' \
+ -e s,^,./${filen}:,g \
+ >> "${tmpfile}"
+ fi
done
# at this point, tmpfile should be in the same format
@@ -782,8 +725,8 @@ function testpatch_usage
echo "--branch=<ref> Forcibly set the branch"
echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'master')"
echo "--build-native=<bool> If true, then build native components (default 'true')"
- echo "--build-tool=<tool> Pick which build tool to focus around (maven, ant)"
- echo "--bugcomments=<bug> Only write comments to the screen and this comma delimited list"
+ echo "--build-tool=<tool> Pick which build tool to focus around (${BUILDTOOLS})"
+ echo "--bugcomments=<bug> Only write comments to the screen and this comma delimited list (${BUGSYSTEMS})"
echo "--contrib-guide=<url> URL to point new users towards project conventions. (default: ${HOW_TO_CONTRIBUTE} )"
echo "--debug If set, then output some extra stuff to stderr"
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
@@ -809,14 +752,12 @@ function testpatch_usage
echo "--test-threads=<int> Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)"
echo ""
echo "Shell binary overrides:"
- echo "--ant-cmd=<cmd> The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')"
echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
echo "--curl-cmd=<cmd> The 'curl' command to use (default 'curl')"
echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')"
echo "--file-cmd=<cmd> The 'file' command to use (default 'file')"
echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
- echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
echo "--sed-cmd=<cmd> The 'sed' command to use (default 'sed')"
@@ -824,12 +765,11 @@ function testpatch_usage
echo "Jenkins-only options:"
echo "--jenkins Run by Jenkins (runs tests and posts results to JIRA)"
echo "--build-url Set the build location web page"
- echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup."
importplugins
- for plugin in ${PLUGINS} ${BUGSYSTEMS} ${TESTFORMATS}; do
+ for plugin in ${BUILDTOOLS} ${PLUGINS} ${BUGSYSTEMS} ${TESTFORMATS}; do
if declare -f ${plugin}_usage >/dev/null 2>&1; then
echo
"${plugin}_usage"
@@ -851,9 +791,6 @@ function parse_args
for i in "$@"; do
case ${i} in
- --ant-cmd=*)
- ANT=${i#*=}
- ;;
--awk-cmd=*)
AWK=${i#*=}
;;
@@ -903,9 +840,6 @@ function parse_args
--dockermode)
DOCKERMODE=true
;;
- --eclipse-home=*)
- ECLIPSE_HOME=${i#*=}
- ;;
--file-cmd=*)
FILE=${i#*=}
;;
@@ -945,9 +879,6 @@ function parse_args
JDK_TEST_LIST=${JDK_TEST_LIST//,/ }
yetus_debug "Multi-JVM test list: ${JDK_TEST_LIST}"
;;
- --mvn-cmd=*)
- MVN=${i#*=}
- ;;
--mv-patch-dir)
RELOCATE_PATCH_DIR=true;
;;
@@ -1036,12 +967,6 @@ function parse_args
add_vote_table 0 reexec "docker mode."
fi
- # if we requested offline, pass that to mvn
- if [[ ${OFFLINE} == "true" ]]; then
- MAVEN_ARGS=(${MAVEN_ARGS[@]} --offline)
- ANT_ARGS=(${ANT_ARGS[@]} -Doffline=)
- fi
-
if [[ -z "${PATCH_OR_ISSUE}" ]]; then
testpatch_usage
exit 1
@@ -1075,8 +1000,6 @@ function parse_args
echo "Running in Jenkins mode"
ISSUE=${PATCH_OR_ISSUE}
RESETREPO=true
- # shellcheck disable=SC2034
- ECLIPSE_PROPERTY="-Declipse.home=${ECLIPSE_HOME}"
else
if [[ ${RESETREPO} == "true" ]] ; then
echo "Running in destructive (--resetrepo) developer mode"
@@ -1194,19 +1117,13 @@ function find_changed_modules
local dir
local buildfile
- case ${BUILDTOOL} in
- maven)
- buildfile=pom.xml
- ;;
- ant)
- buildfile=build.xml
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- bugsystem_finalreport 1
- cleanup_and_exit 1
- ;;
- esac
+ buildfile=$("${BUILDTOOL}_buildfile")
+
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: Unsupported build tool."
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
+ fi
changed_dirs=$(for i in ${CHANGED_FILES}; do dirname "${i}"; done | sort -u)
@@ -1218,7 +1135,7 @@ function find_changed_modules
continue
fi
- builddir=$(find_buildfile_dir ${buildfile} "${i}")
+ builddir=$(find_buildfile_dir "${buildfile}" "${i}")
if [[ -z ${builddir} ]]; then
yetus_error "ERROR: ${buildfile} is not found. Make sure the target is a ${BUILDTOOL}-based project."
bugsystem_finalreport 1
@@ -1232,7 +1149,6 @@ function find_changed_modules
#shellcheck disable=SC2086,SC2116
CHANGED_UNFILTERED_MODULES=$(echo ${CHANGED_UNFILTERED_MODULES})
-
if [[ ${BUILDTOOL} = maven ]]; then
# Filter out modules without code
for module in ${builddirs}; do
@@ -1283,7 +1199,7 @@ function find_changed_modules
fi
yetus_debug "Finding union of ${builddir}"
- builddir=$(find_buildfile_dir ${buildfile} "${builddir}" || true)
+ builddir=$(find_buildfile_dir "${buildfile}" "${builddir}" || true)
#shellcheck disable=SC2034
CHANGED_UNION_MODULES="${builddir}"
@@ -1536,7 +1452,24 @@ function add_test
NEEDED_TESTS=${testname}
elif [[ ! ${NEEDED_TESTS} =~ ${testname} ]] ; then
yetus_debug "Adding ${testname}"
- NEEDED_TESTS="${NEEDED_TESTS} ${testname}"
+ NEEDED_TESTS="${NEEDED_TESTS} ${testname} "
+ fi
+}
+
+## @description Remove the given test type
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param test
+function delete_test
+{
+ local testname=$1
+
+ yetus_debug "Testing against ${testname}"
+
+ if [[ ${NEEDED_TESTS} =~ ${testname} ]] ; then
+ yetus_debug "Removing ${testname}"
+ NEEDED_TESTS="${NEEDED_TESTS// ${testname} }"
fi
}
@@ -1697,7 +1630,7 @@ function verify_patch_file
## @return exit on failure
function apply_patch_file
{
- big_console_header "Applying patch"
+ big_console_header "Applying patch to ${PATCH_BRANCH}"
export PATCH
"${BINDIR}/smart-apply-patch.sh" "${PATCH_DIR}/patch"
@@ -1912,6 +1845,7 @@ function modules_reset
MODULE_STATUS_TIMER=()
MODULE_STATUS_MSG=()
MODULE_STATUS_LOG=()
+ MODULE_COMPILE_LOG=()
}
## @description Utility to print standard module errors
@@ -2023,16 +1957,13 @@ function module_status
MODULE_STATUS_MSG[${index}]="${*}"
else
yetus_error "ASSERT: module_status given bad index: ${index}"
- local frame=0
- while caller $frame; do
- ((frame++));
- done
- echo "$*"
+ yetus_error "ASSERT: module_stats $*"
+ generate_stack
exit 1
fi
}
-## @description run the maven tests for the queued modules
+## @description run the tests for the queued modules
## @audience public
## @stability evolving
## @replaceable no
@@ -2078,7 +2009,9 @@ function modules_workers
fn=$(module_file_fragment "${MODULE[${modindex}]}")
fn="${fn}${jdk}"
modulesuffix=$(basename "${MODULE[${modindex}]}")
- pushd "${BASEDIR}/${MODULE[${modindex}]}" >/dev/null
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ pushd "${BASEDIR}/${MODULE[${modindex}]}" >/dev/null
+ fi
if [[ ${modulesuffix} == . ]]; then
modulesuffix="root"
@@ -2090,26 +2023,11 @@ function modules_workers
continue
fi
- case ${BUILDTOOL} in
- maven)
- #shellcheck disable=SC2086
- echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \
- ${MVN} "${MAVEN_ARGS[@]}" \
- "${@//@@@MODULEFN@@@/${fn}}" \
- ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} -Ptest-patch
- ;;
- ant)
- #shellcheck disable=SC2086
- echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \
- "${ANT}" "${ANT_ARGS[@]}" \
- ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} \
- "${@//@@@MODULEFN@@@/${fn}}"
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- return 1
- ;;
- esac
+ # shellcheck disable=2086,2046
+ echo_and_redirect "${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt" \
+ $("${BUILDTOOL}_executor") \
+ ${MODULEEXTRAPARAM[${modindex}]//@@@MODULEFN@@@/${fn}} \
+ "${@//@@@MODULEFN@@@/${fn}}"
if [[ $? == 0 ]] ; then
module_status \
@@ -2125,11 +2043,20 @@ function modules_workers
"${modulesuffix} in ${repo} failed${statusjdk}."
((result = result + 1))
fi
+
+ # compile is special
+ if [[ ${testtype} = compile ]]; then
+ MODULE_COMPILE_LOG[${modindex}]="${PATCH_DIR}/${repostatus}-${testtype}-${fn}.txt"
+ yetus_debug "Comile log set to ${MODULE_COMPILE_LOG[${modindex}]}"
+ fi
+
savestop=$(stop_clock)
MODULE_STATUS_TIMER[${modindex}]=${savestop}
# shellcheck disable=SC2086
echo "Elapsed: $(clock_display ${savestop})"
- popd >/dev/null
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ popd >/dev/null
+ fi
((modindex=modindex+1))
done
@@ -2169,85 +2096,59 @@ function personality_enqueue_module
((MODCOUNT=MODCOUNT+1))
}
-## @description Confirm compilation pre-patch
+## @description Utility to push many tests into the failure list
## @audience private
-## @stability stable
+## @stability evolving
## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_javac
+## @param testdesc
+## @param testlist
+function populate_test_table
{
- local result=0
- local -r savejavahome=${JAVA_HOME}
- local multijdkmode=false
- local jdkindex=0
-
- big_console_header "Pre-patch ${PATCH_BRANCH} javac compilation"
-
- verify_needed_test javac
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need javac tests."
- return 0
- fi
-
- verify_multijdk_test javac
- if [[ $? == 1 ]]; then
- multijdkmode=true
- fi
+ local reason=$1
+ shift
+ local first=""
+ local i
- for jdkindex in ${JDK_DIR_LIST}; do
- if [[ ${multijdkmode} == true ]]; then
- JAVA_HOME=${jdkindex}
+ for i in "$@"; do
+ if [[ -z "${first}" ]]; then
+ add_test_table "${reason}" "${i}"
+ first="${reason}"
+ else
+ add_test_table " " "${i}"
fi
-
- personality_modules branch javac
- case ${BUILDTOOL} in
- maven)
- modules_workers branch javac clean test-compile
- ;;
- ant)
- modules_workers branch javac
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- return 1
- ;;
- esac
-
- ((result=result + $?))
- modules_messages branch javac true
-
done
- JAVA_HOME=${savejavahome}
-
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
- return 0
}
-## @description Confirm Javadoc pre-patch
+## @description Run and verify the output of the appropriate unit tests
## @audience private
-## @stability stable
+## @stability evolving
## @replaceable no
## @return 0 on success
## @return 1 on failure
-function precheck_javadoc
+function check_unittests
{
+ local i
+ local testsys
+ local test_logfile
local result=0
local -r savejavahome=${JAVA_HOME}
local multijdkmode=false
+ local jdk=""
local jdkindex=0
+ local statusjdk
+ local formatresult=0
+ local needlog
+ local unitlogs
- big_console_header "Pre-patch ${PATCH_BRANCH} Javadoc verification"
+ verify_needed_test unit
- verify_needed_test javadoc
if [[ $? == 0 ]]; then
- echo "Patch does not appear to need javadoc tests."
- return 0
+ return 0
fi
- verify_multijdk_test javadoc
+ big_console_header "Running unit tests"
+
+ verify_multijdk_test unit
if [[ $? == 1 ]]; then
multijdkmode=true
fi
@@ -2255,757 +2156,88 @@ function precheck_javadoc
for jdkindex in ${JDK_DIR_LIST}; do
if [[ ${multijdkmode} == true ]]; then
JAVA_HOME=${jdkindex}
+ jdk=$(report_jvm_version "${JAVA_HOME}")
+ statusjdk="JDK v${jdk} "
+ jdk="-jdk${jdk}"
+ jdk=${jdk// /}
fi
- personality_modules branch javadoc
- case ${BUILDTOOL} in
- maven)
- modules_workers branch javadoc clean javadoc:javadoc
- ;;
- ant)
- modules_workers branch javadoc clean javadoc
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- return 1
- ;;
- esac
+ personality_modules patch unit
+ "${BUILDTOOL}_modules_worker" patch unit
- ((result=result + $?))
- modules_messages branch javadoc true
+ ((result=result+$?))
+
+ modules_messages patch unit false
+ if [[ ${result} == 0 ]]; then
+ continue
+ fi
+
+ i=0
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ module=${MODULE[${i}]}
+ fn=$(module_file_fragment "${module}")
+ fn="${fn}${jdk}"
+ test_logfile="${PATCH_DIR}/patch-unit-${fn}.txt"
+
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ pushd "${MODULE[${i}]}" >/dev/null
+ fi
+
+ needlog=0
+ for testsys in ${TESTFORMATS}; do
+ if declare -f ${testsys}_process_tests >/dev/null; then
+ yetus_debug "Calling ${testsys}_process_tests"
+ "${testsys}_process_tests" "${module}" "${test_logfile}" "${fn}"
+ formatresult=$?
+ ((result=result+formatresult))
+ if [[ "${formatresult}" != 0 ]]; then
+ needlog=1
+ fi
+ fi
+ done
+
+ if [[ ${needlog} == 1 ]]; then
+ unitlogs="${unitlogs} @@BASE@@/patch-unit-${fn}.txt"
+ fi
+
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ popd >/dev/null
+ fi
+
+ ((i=i+1))
+ done
done
JAVA_HOME=${savejavahome}
- if [[ ${result} -gt 0 ]]; then
- return 1
+ if [[ -n "${unitlogs}" ]]; then
+ add_footer_table "unit test logs" "${unitlogs}"
fi
- return 0
-}
-
-## @description Confirm site pre-patch
-## @audience private
-## @stability stable
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_site
-{
- local result=0
- if [[ ${BUILDTOOL} != maven ]]; then
- return 0
+ if [[ ${JENKINS} == true ]]; then
+ add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"
fi
- big_console_header "Pre-patch ${PATCH_BRANCH} site verification"
-
- verify_needed_test site
- if [[ $? == 0 ]];then
- echo "Patch does not appear to need site tests."
- return 0
- fi
+ for testsys in ${TESTFORMATS}; do
+ if declare -f ${testsys}_finalize_results >/dev/null; then
+ yetus_debug "Calling ${testsys}_finalize_results"
+ "${testsys}_finalize_results" "${statusjdk}"
+ fi
+ done
- personality_modules branch site
- modules_workers branch site clean site site:stage
- result=$?
- modules_messages branch site true
- if [[ ${result} != 0 ]]; then
+ if [[ ${result} -gt 0 ]]; then
return 1
fi
return 0
}
-## @description Confirm the source environment pre-patch
-## @audience private
-## @stability stable
+## @description Write comments onto bug systems that have code review support.
+## @description File should be in the form of "file:line:comment"
+## @audience public
+## @stability evolving
## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_without_patch
-{
- local result=0
-
- precheck_mvninstall
-
- if [[ $? -gt 0 ]]; then
- ((result = result +1 ))
- fi
-
- precheck_javac
-
- if [[ $? -gt 0 ]]; then
- ((result = result +1 ))
- fi
-
- precheck_javadoc
-
- if [[ $? -gt 0 ]]; then
- ((result = result +1 ))
- fi
-
- precheck_site
-
- if [[ $? -gt 0 ]]; then
- ((result = result +1 ))
- fi
-
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
-
- return 0
-}
-
-## @description Check the current directory for @author tags
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_author
-{
- local authorTags
- local -r appname=$(basename "${BASH_SOURCE-$0}")
-
- big_console_header "Checking there are no @author tags in the patch."
-
- start_clock
-
- if [[ ${CHANGED_FILES} =~ ${appname} ]]; then
- echo "Skipping @author checks as ${appname} has been patched."
- add_vote_table 0 @author "Skipping @author checks as ${appname} has been patched."
- return 0
- fi
-
- authorTags=$("${GREP}" -c -i '^[^-].*@author' "${PATCH_DIR}/patch")
- echo "There appear to be ${authorTags} @author tags in the patch."
- if [[ ${authorTags} != 0 ]] ; then
- add_vote_table -1 @author \
- "The patch appears to contain ${authorTags} @author tags which the" \
- " community has agreed to not allow in code contributions."
- return 1
- fi
- add_vote_table +1 @author "The patch does not contain any @author tags."
- return 0
-}
-
-## @description Check the patch file for changed/new tests
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_modified_unittests
-{
- local testReferences=0
- local i
-
- big_console_header "Checking there are new or changed tests in the patch."
-
- verify_needed_test unit
-
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need new or modified tests."
- return 0
- fi
-
- start_clock
-
- for i in ${CHANGED_FILES}; do
- if [[ ${i} =~ (^|/)test/ ]]; then
- ((testReferences=testReferences + 1))
- fi
- done
-
- echo "There appear to be ${testReferences} test file(s) referenced in the patch."
- if [[ ${testReferences} == 0 ]] ; then
- add_vote_table -1 "test4tests" \
- "The patch doesn't appear to include any new or modified tests. " \
- "Please justify why no new tests are needed for this patch." \
- "Also please list what manual steps were performed to verify this patch."
- return 1
- fi
- add_vote_table +1 "test4tests" \
- "The patch appears to include ${testReferences} new or modified test files."
- return 0
-}
-
-## @description Helper for check_patch_javac
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function count_javac_probs
-{
- local warningfile=$1
- local val1
- local val2
-
- case ${BUILDTOOL} in
- maven)
- #shellcheck disable=SC2016,SC2046
- ${GREP} '\[WARNING\]' "${warningfile}" | ${AWK} '{sum+=1} END {print sum}'
- ;;
- ant)
- #shellcheck disable=SC2016
- val1=$(${GREP} -E "\[javac\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
- #shellcheck disable=SC2016
- val2=$(${GREP} -E "\[javac\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
- echo $((val1+val2))
- ;;
- esac
-}
-
-## @description Count and compare the number of javac warnings pre- and post- patch
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_patch_javac
-{
- local i
- local result=0
- local fn
- local -r savejavahome=${JAVA_HOME}
- local multijdkmode=false
- local jdk=""
- local jdkindex=0
- local statusjdk
- declare -i numbranch=0
- declare -i numpatch=0
-
- big_console_header "Determining number of patched javac errors"
-
- verify_needed_test javac
-
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need javac tests."
- return 0
- fi
-
- verify_multijdk_test javac
- if [[ $? == 1 ]]; then
- multijdkmode=true
- fi
-
- for jdkindex in ${JDK_DIR_LIST}; do
- if [[ ${multijdkmode} == true ]]; then
- JAVA_HOME=${jdkindex}
- jdk=$(report_jvm_version "${JAVA_HOME}")
- yetus_debug "Using ${JAVA_HOME} to run this set of tests"
- statusjdk=" with JDK v${jdk}"
- jdk="-jdk${jdk}"
- jdk=${jdk// /}
- fi
-
- personality_modules patch javac
-
- case ${BUILDTOOL} in
- maven)
- modules_workers patch javac clean test-compile
- ;;
- ant)
- modules_workers patch javac
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- return 1
- ;;
- esac
-
- i=0
- until [[ ${i} -eq ${#MODULE[@]} ]]; do
- if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
- ((result=result+1))
- ((i=i+1))
- continue
- fi
-
- fn=$(module_file_fragment "${MODULE[${i}]}")
- fn="${fn}${jdk}"
- module_suffix=$(basename "${MODULE[${i}]}")
- if [[ ${module_suffix} == \. ]]; then
- module_suffix=root
- fi
-
- # if it was a new module, this won't exist.
- if [[ -f "${PATCH_DIR}/branch-javac-${fn}.txt" ]]; then
- ${GREP} -i warning "${PATCH_DIR}/branch-javac-${fn}.txt" \
- > "${PATCH_DIR}/branch-javac-${fn}-warning.txt"
- else
- touch "${PATCH_DIR}/branch-javac-${fn}.txt" \
- "${PATCH_DIR}/branch-javac-${fn}-warning.txt"
- fi
-
- if [[ -f "${PATCH_DIR}/patch-javac-${fn}.txt" ]]; then
- ${GREP} -i warning "${PATCH_DIR}/patch-javac-${fn}.txt" \
- > "${PATCH_DIR}/patch-javac-${fn}-warning.txt"
- else
- touch "${PATCH_DIR}/patch-javac-${fn}.txt" \
- "${PATCH_DIR}/patch-javac-${fn}-warning.txt"
- fi
-
- numbranch=$(count_javac_probs "${PATCH_DIR}/branch-javac-${fn}-warning.txt")
- numpatch=$(count_javac_probs "${PATCH_DIR}/patch-javac-${fn}-warning.txt")
-
- if [[ -n ${numbranch}
- && -n ${numpatch}
- && ${numpatch} -gt ${numbranch} ]]; then
-
- ${DIFF} -u "${PATCH_DIR}/branch-javac-${fn}-warning.txt" \
- "${PATCH_DIR}/patch-javac-${fn}-warning.txt" \
- > "${PATCH_DIR}/javac-${fn}-diff.txt"
-
- module_status ${i} -1 "javac-${fn}-diff.txt" \
- "Patched ${module_suffix} generated "\
- "$((numpatch-numbranch)) additional warning messages${statusjdk}." \
-
- ((result=result+1))
- fi
- ((i=i+1))
- done
-
- modules_messages patch javac true
- done
- JAVA_HOME=${savejavahome}
-
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Helper for check_patch_javadoc
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function count_javadoc_probs
-{
- local warningfile=$1
- local val1
- local val2
-
- case ${BUILDTOOL} in
- maven)
- #shellcheck disable=SC2016,SC2046
- ${GREP} -E "^[0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}'
- ;;
- ant)
- #shellcheck disable=SC2016
- val1=$(${GREP} -E "\[javadoc\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
- #shellcheck disable=SC2016
- val2=$(${GREP} -E "\[javadoc\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
- echo $((val1+val2))
- ;;
- esac
-}
-
-## @description Count and compare the number of JavaDoc warnings pre- and post- patch
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_patch_javadoc
-{
- local i
- local result=0
- local fn
- local -r savejavahome=${JAVA_HOME}
- local multijdkmode=false
- local jdk=""
- local jdkindex=0
- local statusjdk
- declare -i numbranch=0
- declare -i numpatch=0
-
- big_console_header "Determining number of patched javadoc warnings"
-
- verify_needed_test javadoc
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need javadoc tests."
- return 0
- fi
-
- verify_multijdk_test javadoc
- if [[ $? == 1 ]]; then
- multijdkmode=true
- fi
-
- for jdkindex in ${JDK_DIR_LIST}; do
- if [[ ${multijdkmode} == true ]]; then
- JAVA_HOME=${jdkindex}
- jdk=$(report_jvm_version "${JAVA_HOME}")
- yetus_debug "Using ${JAVA_HOME} to run this set of tests"
- statusjdk=" with JDK v${jdk}"
- jdk="-jdk${jdk}"
- jdk=${jdk// /}
- fi
-
- personality_modules patch javadoc
- case ${BUILDTOOL} in
- maven)
- modules_workers patch javadoc clean javadoc:javadoc
- ;;
- ant)
- modules_workers patch javadoc clean javadoc
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- return 1
- ;;
- esac
-
- i=0
- until [[ ${i} -eq ${#MODULE[@]} ]]; do
- if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
- ((result=result+1))
- ((i=i+1))
- continue
- fi
-
- fn=$(module_file_fragment "${MODULE[${i}]}")
- fn="${fn}${jdk}"
- module_suffix=$(basename "${MODULE[${i}]}")
- if [[ ${module_suffix} == \. ]]; then
- module_suffix=root
- fi
-
- if [[ -f "${PATCH_DIR}/branch-javadoc-${fn}.txt" ]]; then
- ${GREP} -i warning "${PATCH_DIR}/branch-javadoc-${fn}.txt" \
- > "${PATCH_DIR}/branch-javadoc-${fn}-warning.txt"
- else
- touch "${PATCH_DIR}/branch-javadoc-${fn}.txt" \
- "${PATCH_DIR}/branch-javadoc-${fn}-warning.txt"
- fi
-
- if [[ -f "${PATCH_DIR}/patch-javadoc-${fn}.txt" ]]; then
- ${GREP} -i warning "${PATCH_DIR}/patch-javadoc-${fn}.txt" \
- > "${PATCH_DIR}/patch-javadoc-${fn}-warning.txt"
- else
- touch "${PATCH_DIR}/patch-javadoc-${fn}.txt" \
- "${PATCH_DIR}/patch-javadoc-${fn}-warning.txt"
- fi
-
- numbranch=$(count_javadoc_probs "${PATCH_DIR}/branch-javadoc-${fn}.txt")
- numpatch=$(count_javadoc_probs "${PATCH_DIR}/patch-javadoc-${fn}.txt")
-
- if [[ -n ${numbranch}
- && -n ${numpatch}
- && ${numpatch} -gt ${numbranch} ]] ; then
-
- ${DIFF} -u "${PATCH_DIR}/branch-javadoc-${fn}-warning.txt" \
- "${PATCH_DIR}/patch-javadoc-${fn}-warning.txt" \
- > "${PATCH_DIR}/javadoc-${fn}-diff.txt"
-
- module_status ${i} -1 "javadoc-${fn}-diff.txt" \
- "Patched ${module_suffix} generated "\
- "$((numpatch-numbranch)) additional warning messages${statusjdk}."
-
- ((result=result+1))
- fi
- ((i=i+1))
- done
-
- modules_messages patch javadoc true
- done
- JAVA_HOME=${savejavahome}
-
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Make sure site still compiles
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_site
-{
- local result=0
-
- if [[ ${BUILDTOOL} != maven ]]; then
- return 0
- fi
-
- big_console_header "Determining number of patched site errors"
-
- verify_needed_test site
- if [[ $? == 0 ]]; then
- echo "Patch does not appear to need site tests."
- return 0
- fi
-
- personality_modules patch site
- modules_workers patch site clean site site:stage -Dmaven.javadoc.skip=true
- result=$?
- modules_messages patch site true
- if [[ ${result} != 0 ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Verify mvn install works
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function precheck_mvninstall
-{
- local result=0
-
- if [[ ${BUILDTOOL} != maven ]]; then
- return 0
- fi
-
- big_console_header "Verifying mvn install works"
-
- verify_needed_test javadoc
- result=$?
-
- verify_needed_test javac
- ((result = result + $? ))
- if [[ ${result} == 0 ]]; then
- echo "This patch does not appear to need mvn install checks."
- return 0
- fi
-
- personality_modules branch mvninstall
- modules_workers branch mvninstall -fae clean install -Dmaven.javadoc.skip=true
- result=$?
- modules_messages branch mvninstall true
- if [[ ${result} != 0 ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Verify mvn install works
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_mvninstall
-{
- local result=0
-
- if [[ ${BUILDTOOL} != maven ]]; then
- return 0
- fi
-
- big_console_header "Verifying mvn install still works"
-
- verify_needed_test javadoc
- result=$?
-
- verify_needed_test javac
- ((result = result + $? ))
- if [[ ${result} == 0 ]]; then
- echo "This patch does not appear to need mvn install checks."
- return 0
- fi
-
- personality_modules patch mvninstall
- modules_workers patch mvninstall clean install -Dmaven.javadoc.skip=true
- result=$?
- modules_messages patch mvninstall true
- if [[ ${result} != 0 ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Make sure Maven's eclipse generation works.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_mvn_eclipse
-{
- local result=0
-
- if [[ ${BUILDTOOL} != maven ]]; then
- return 0
- fi
-
- big_console_header "Verifying mvn eclipse:eclipse still works"
-
- verify_needed_test javac
- if [[ $? == 0 ]]; then
- echo "Patch does not touch any java files. Skipping mvn eclipse:eclipse"
- return 0
- fi
-
- personality_modules patch eclipse
- modules_workers patch eclipse eclipse:eclipse
- result=$?
- modules_messages patch eclipse true
- if [[ ${result} != 0 ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Utility to push many tests into the failure list
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param testdesc
-## @param testlist
-function populate_test_table
-{
- local reason=$1
- shift
- local first=""
- local i
-
- for i in "$@"; do
- if [[ -z "${first}" ]]; then
- add_test_table "${reason}" "${i}"
- first="${reason}"
- else
- add_test_table " " "${i}"
- fi
- done
-}
-
-## @description Run and verify the output of the appropriate unit tests
-## @audience private
-## @stability evolving
-## @replaceable no
-## @return 0 on success
-## @return 1 on failure
-function check_unittests
-{
- local i
- local testsys
- local test_logfile
- local result=0
- local -r savejavahome=${JAVA_HOME}
- local multijdkmode=false
- local jdk=""
- local jdkindex=0
- local statusjdk
- local formatresult=0
- local needlog
- local unitlogs
-
- big_console_header "Running unit tests"
-
- verify_needed_test unit
-
- if [[ $? == 0 ]]; then
- echo "Existing unit tests do not test patched files. Skipping."
- return 0
- fi
-
- verify_multijdk_test unit
- if [[ $? == 1 ]]; then
- multijdkmode=true
- fi
-
- for jdkindex in ${JDK_DIR_LIST}; do
- if [[ ${multijdkmode} == true ]]; then
- JAVA_HOME=${jdkindex}
- jdk=$(report_jvm_version "${JAVA_HOME}")
- statusjdk="JDK v${jdk} "
- jdk="-jdk${jdk}"
- jdk=${jdk// /}
- fi
-
- personality_modules patch unit
- case ${BUILDTOOL} in
- maven)
- modules_workers patch unit clean test -fae
- ;;
- ant)
- modules_workers patch unit
- ;;
- *)
- yetus_error "ERROR: Unsupported build tool."
- return 1
- ;;
- esac
- ((result=result+$?))
-
- modules_messages patch unit false
- if [[ ${result} == 0 ]]; then
- continue
- fi
-
- i=0
- until [[ $i -eq ${#MODULE[@]} ]]; do
- module=${MODULE[${i}]}
- fn=$(module_file_fragment "${module}")
- fn="${fn}${jdk}"
- test_logfile="${PATCH_DIR}/patch-unit-${fn}.txt"
-
- pushd "${MODULE[${i}]}" >/dev/null
-
- needlog=0
- for testsys in ${TESTFORMATS}; do
- if declare -f ${testsys}_process_tests >/dev/null; then
- yetus_debug "Calling ${testsys}_process_tests"
- "${testsys}_process_tests" "${module}" "${test_logfile}" "${fn}"
- formatresult=$?
- ((result=result+formatresult))
- if [[ "${formatresult}" != 0 ]]; then
- needlog=1
- fi
- fi
- done
-
- if [[ ${needlog} == 1 ]]; then
- unitlogs="${unitlogs} @@BASE@@/patch-unit-${fn}.txt"
- fi
-
- popd >/dev/null
-
- ((i=i+1))
- done
-
- done
- JAVA_HOME=${savejavahome}
-
- if [[ -n "${unitlogs}" ]]; then
- add_footer_table "unit test logs" "${unitlogs}"
- fi
-
- if [[ ${JENKINS} == true ]]; then
- add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"
- fi
-
- for testsys in ${TESTFORMATS}; do
- if declare -f ${testsys}_finalize_results >/dev/null; then
- yetus_debug "Calling ${testsys}_finalize_results"
- "${testsys}_finalize_results" "${statusjdk}"
- fi
- done
-
- if [[ ${result} -gt 0 ]]; then
- return 1
- fi
- return 0
-}
-
-## @description Write comments onto bug systems that have code review support.
-## @description File should be in the form of "file:line:comment"
-## @audience public
-## @stability evolving
-## @replaceable no
-## @param filename
-function bugsystem_linecomments
+## @param filename
+function bugsystem_linecomments
{
declare title=$1
declare fn=$2
@@ -3077,372 +2309,694 @@ function cleanup_and_exit
exit ${result}
}
-## @description Driver to execute _postcheckout routines
+## @description Driver to execute _tests routines
## @audience private
## @stability evolving
## @replaceable no
-function postcheckout
+function runtests
{
- local routine
local plugin
- for routine in find_java_home verify_patch_file; do
- verify_patchdir_still_exists
+ ### Run tests for Jenkins or if explictly asked for by a developer
+ if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then
- yetus_debug "Running ${routine}"
- ${routine}
+ verify_patchdir_still_exists
+ check_unittests
(( RESULT = RESULT + $? ))
- if [[ ${RESULT} != 0 ]] ; then
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
- done
+ fi
for plugin in ${PLUGINS}; do
verify_patchdir_still_exists
-
- if declare -f ${plugin}_postcheckout >/dev/null 2>&1; then
-
- yetus_debug "Running ${plugin}_postcheckout"
+ if declare -f ${plugin}_tests >/dev/null 2>&1; then
+ modules_reset
+ yetus_debug "Running ${plugin}_tests"
#shellcheck disable=SC2086
- ${plugin}_postcheckout
-
+ ${plugin}_tests
(( RESULT = RESULT + $? ))
- if [[ ${RESULT} != 0 ]] ; then
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
fi
done
}
-## @description Driver to execute _preapply routines
+## @description Import content from test-patch.d and optionally
+## @description from user provided plugin directory
## @audience private
## @stability evolving
## @replaceable no
-function preapply
+function importplugins
{
- local routine
- local plugin
+ local i
+ local files=()
- for routine in precheck_without_patch check_author \
- check_modified_unittests
- do
- verify_patchdir_still_exists
+ if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
+ if [[ -d "${BINDIR}/test-patch.d" ]]; then
+ files=(${BINDIR}/test-patch.d/*.sh)
+ fi
+ fi
- yetus_debug "Running ${routine}"
- ${routine}
+ if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
+ yetus_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
+ files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
+ fi
- (( RESULT = RESULT + $? ))
+ for i in "${files[@]}"; do
+ if [[ -f ${i} ]]; then
+ yetus_debug "Importing ${i}"
+ . "${i}"
+ fi
done
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
+ if [[ -z ${PERSONALITY}
+ && -f "${BINDIR}/personality/${PROJECT_NAME}.sh" ]]; then
+ PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
+ fi
- if declare -f ${plugin}_preapply >/dev/null 2>&1; then
+ if [[ -n ${PERSONALITY} ]]; then
+ if [[ ! -f ${PERSONALITY} ]]; then
+ if [[ -f "${BINDIR}/personality/${PROJECT_NAME}.sh" ]]; then
+ PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
+ else
+ yetus_debug "Can't find ${PERSONALITY} to import."
+ return
+ fi
+ fi
+ yetus_debug "Importing ${PERSONALITY}"
+ . "${PERSONALITY}"
+ fi
+}
- yetus_debug "Running ${plugin}_preapply"
+## @description Let plugins also get a copy of the arguments
+## @audience private
+## @stability evolving
+## @replaceable no
+function parse_args_plugins
+{
+ for plugin in ${PLUGINS} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOLS}; do
+ if declare -f ${plugin}_parse_args >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_parse_args"
#shellcheck disable=SC2086
- ${plugin}_preapply
-
+ ${plugin}_parse_args "$@"
(( RESULT = RESULT + $? ))
fi
done
+
+ BUGCOMMENTS=${BUGCOMMENTS:-${BUGSYSTEMS}}
+ if [[ ! ${BUGCOMMENTS} =~ console ]]; then
+ BUGCOMMENTS="${BUGCOMMENTS} console"
+ fi
+
+ BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}}
}
-## @description Driver to execute _postapply routines
+## @description Let plugins also get a copy of the arguments
## @audience private
## @stability evolving
## @replaceable no
-function postapply
+function plugins_initialize
{
- local routine
- local plugin
- local retval
+ declare plugin
+
+ for plugin in ${PLUGINS} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOLS}; do
+ if declare -f ${plugin}_initialize >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_initialize"
+ #shellcheck disable=SC2086
+ ${plugin}_initialize
+ (( RESULT = RESULT + $? ))
+ fi
+ done
+}
+
+## @description Register test-patch.d plugins
+## @audience public
+## @stability stable
+## @replaceable no
+function add_plugin
+{
+ PLUGINS="${PLUGINS} $1"
+}
+
+## @description Register test-patch.d bugsystems
+## @audience public
+## @stability stable
+## @replaceable no
+function add_bugsystem
+{
+ BUGSYSTEMS="${BUGSYSTEMS} $1"
+}
+
+## @description Register test-patch.d test output formats
+## @audience public
+## @stability stable
+## @replaceable no
+function add_test_format
+{
+ TESTFORMATS="${TESTFORMATS} $1"
+}
+
+## @description Register test-patch.d build tools
+## @audience public
+## @stability stable
+## @replaceable no
+function add_build_tool
+{
+ BUILDTOOLS="${BUILDTOOLS} $1"
+}
+
+## @description Calculate the differences between the specified files
+## @description and output it to stdout.
+## @audience public
+## @stability evolving
+## @replaceable no
+function calcdiffs
+{
+ local orig=$1
+ local new=$2
+ local tmp=${PATCH_DIR}/pl.$$.${RANDOM}
+ local count=0
+ local j
+
+ # first, pull out just the errors
+ # shellcheck disable=SC2016
+ ${AWK} -F: '{print $NF}' "${orig}" > "${tmp}.branch"
+
+ # shellcheck disable=SC2016
+ ${AWK} -F: '{print $NF}' "${new}" > "${tmp}.patch"
+
+ # compare the errors, generating a string of line
+ # numbers. Sorry portability: GNU diff makes this too easy
+ ${DIFF} --unchanged-line-format="" \
+ --old-line-format="" \
+ --new-line-format="%dn " \
+ "${tmp}.branch" \
+ "${tmp}.patch" > "${tmp}.lined"
+
+ # now, pull out those lines of the raw output
+ # shellcheck disable=SC2013
+ for j in $(cat "${tmp}.lined"); do
+ # shellcheck disable=SC2086
+ head -${j} "${new}" | tail -1
+ done
+
+ rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
+}
+
+## @description Helper routine for plugins to ask projects, etc
+## @description to count problems in a log file
+## @description and output it to stdout.
+## @audience public
+## @stability evolving
+## @replaceable no
+## @return number of issues
+function generic_count_probs
+{
+ declare testtype=$1
+ declare input=$2
+
+ if declare -f ${PROJECT}_${testtype}_count_probs >/dev/null; then
+ "${PROJECT}_${testtype}_count_probs" "${input}"
+ elif declare -f ${BUILDTOOL}_${testtype}_count_probs >/dev/null; then
+ "${BUILDTOOL}_${testtype}_count_probs" "${input}"
+ else
+ yetus_error "ERROR: ${testtype}: No function defined to count problems."
+ echo 0
+ fi
+}
+
+## @description Helper routine for plugins to do a pre-patch prun
+## @audience public
+## @stability evolving
+## @replaceable no
+## @param testype
+## @param multijdk
+## @return 1 on failure
+## @return 0 on success
+function generic_pre_handler
+{
+ declare testtype=$1
+ declare multijdkmode=$2
+ declare result=0
+ declare -r savejavahome=${JAVA_HOME}
+ declare multijdkmode=false
+ declare jdkindex=0
+
+ verify_needed_test "${testtype}"
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "Pre-patch ${testtype} verification on ${PATCH_BRANCH}"
+
+ verify_multijdk_test "${testtype}"
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
+ fi
+
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
+ fi
+
+ personality_modules branch "${testtype}"
+ "${BUILDTOOL}_modules_worker" branch "${testtype}"
+
+ ((result=result + $?))
+ modules_messages branch "${testtype}" true
+
+ done
+ JAVA_HOME=${savejavahome}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Generic post-patch log handler
+## @audience public
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+## @param origlog
+## @param testtype
+## @param multijdkmode
+function generic_postlog_compare
+{
+ declare origlog=$1
+ declare testtype=$2
+ declare multijdk=$3
+ declare result=0
+ declare i
+ declare fn
+ declare jdk
+ declare statusjdk
+
+ if [[ ${multijdk} == true ]]; then
+ jdk=$(report_jvm_version "${JAVA_HOME}")
+ statusjdk=" with JDK v${jdk}"
+ jdk="-jdk${jdk}"
+ jdk=${jdk// /}
+ fi
+
+ i=0
+ until [[ ${i} -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+
+ fn=$(module_file_fragment "${MODULE[${i}]}")
+ fn="${fn}${jdk}"
+ module_suffix=$(basename "${MODULE[${i}]}")
+ if [[ ${module_suffix} == \. ]]; then
+ module_suffix=root
+ fi
+
+ yetus_debug "${testtype}: branch-${origlog}-${fn}.txt vs. patch-${origlog}-${fn}.txt"
+
+ # if it was a new module, this won't exist.
+ if [[ -f "${PATCH_DIR}/branch-${origlog}-${fn}.txt" ]]; then
+ ${GREP} -i warning "${PATCH_DIR}/branch-${origlog}-${fn}.txt" \
+ > "${PATCH_DIR}/branch-${testtype}-${fn}-warning.txt"
+ else
+ touch "${PATCH_DIR}/branch-${origlog}-${fn}.txt" \
+ "${PATCH_DIR}/branch-${testtype}-${fn}-warning.txt"
+ fi
+
+ if [[ -f "${PATCH_DIR}/patch-${origlog}-${fn}.txt" ]]; then
+ ${GREP} -i warning "${PATCH_DIR}/patch-${origlog}-${fn}.txt" \
+ > "${PATCH_DIR}/patch-${testtype}-${fn}-warning.txt"
+ else
+ touch "${PATCH_DIR}/patch-${origlog}-${fn}.txt" \
+ "${PATCH_DIR}/patch-${testtype}-${fn}-warning.txt"
+ fi
- compute_gitdiff
+ numbranch=$("generic_count_probs" "${testtype}" "${PATCH_DIR}/branch-${testtype}-${fn}-warning.txt")
+ numpatch=$("generic_count_probs" "${testtype}" "${PATCH_DIR}/patch-${testtype}-${fn}-warning.txt")
- check_patch_javac
- retval=$?
- if [[ ${retval} -gt 1 ]] ; then
- bugsystem_finalreport 1
- cleanup_and_exit 1
- fi
+ yetus_debug "${testtype}: old: ${numbranch} vs new: ${numpatch}"
- ((RESULT = RESULT + retval))
+ if [[ -n ${numbranch}
+ && -n ${numpatch}
+ && ${numpatch} -gt ${numbranch} ]]; then
- # shellcheck disable=SC2043
- for routine in check_site
- do
- verify_patchdir_still_exists
- yetus_debug "Running ${routine}"
- ${routine}
- (( RESULT = RESULT + $? ))
- done
+ ${DIFF} -u "${PATCH_DIR}/branch-${testtype}-${fn}-warning.txt" \
+ "${PATCH_DIR}/patch-${testtype}-${fn}-warning.txt" \
+ > "${PATCH_DIR}/${testtype}-${fn}-diff.txt"
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_postapply >/dev/null 2>&1; then
- yetus_debug "Running ${plugin}_postapply"
- #shellcheck disable=SC2086
- ${plugin}_postapply
- (( RESULT = RESULT + $? ))
+ add_vote_table -1 "${testtype}" "${fn}${statusjdk} has problems."
+ add_footer_table "${testtype}" "${fn}: @@BASE@@/${testtype}-${fn}-diff.txt"
+
+ ((result=result+1))
fi
+ ((i=i+1))
done
+ modules_messages patch "${testtype}" true
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
}
-## @description Driver to execute _postinstall routines
-## @audience private
+## @description Generic post-patch handler
+## @audience public
## @stability evolving
## @replaceable no
-function postinstall
-{
- local routine
- local plugin
+## @return 0 on success
+## @return 1 on failure
+## @param origlog
+## @param testtype
+## @param multijdkmode
+## @param run commands
+function generic_post_handler
+{
+ declare origlog=$1
+ declare testtype=$2
+ declare multijdkmode=$3
+ declare need2run=$4
+ declare i
+ declare result=0
+ declare fn
+ declare -r savejavahome=${JAVA_HOME}
+ declare jdk=""
+ declare jdkindex=0
+ declare statusjdk
+ declare -i numbranch=0
+ declare -i numpatch=0
- verify_patchdir_still_exists
- for routine in check_patch_javadoc check_mvn_eclipse
- do
- verify_patchdir_still_exists
- yetus_debug "Running ${routine}"
- ${routine}
- (( RESULT = RESULT + $? ))
- done
+ verify_needed_test "${testtype}"
+ if [[ $? == 0 ]]; then
+ yetus_debug "${testtype} not needed"
+ return 0
+ fi
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_postinstall >/dev/null 2>&1; then
- yetus_debug "Running ${plugin}_postinstall"
- #shellcheck disable=SC2086
- ${plugin}_postinstall
- (( RESULT = RESULT + $? ))
+ big_console_header "Patch ${testtype} verification"
+
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
+ yetus_debug "Using ${JAVA_HOME} to run this set of tests"
+ fi
+
+ if [[ ${need2run} = true ]]; then
+ personality_modules "${codebase}" "${testtype}"
+ "${BUILDTOOL}_modules_worker" "${codebase}" "${testtype}"
+
+ if [[ ${UNSUPPORTED_TEST} = true ]]; then
+ return 0
+ fi
fi
+
+ generic_postlog_compare "${origlog}" "${testtype}" "${multijdkmode}"
+ ((result=result+$?))
done
+ JAVA_HOME=${savejavahome}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
}
-## @description Driver to execute _tests routines
-## @audience private
+## @description Execute the compile phase. This will callout
+## @description to _compile
+## @audience public
## @stability evolving
## @replaceable no
-function runtests
+## @param branch|patch
+## @return 0 on success
+## @return 1 on failure
+function compile
{
- local plugin
+ declare codebase=$1
+ declare result=0
+ declare -r savejavahome=${JAVA_HOME}
+ declare multijdkmode=false
+ declare jdkindex=0
- ### Run tests for Jenkins or if explictly asked for by a developer
- if [[ ${JENKINS} == "true" || ${RUN_TESTS} == "true" ]] ; then
+ verify_needed_test compile
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
- verify_patchdir_still_exists
- check_unittests
+ if [[ ${codebase} = "branch" ]]; then
+ big_console_header "Pre-patch ${PATCH_BRANCH} compilation"
+ else
+ big_console_header "Patch compilation"
+ fi
- (( RESULT = RESULT + $? ))
+ verify_multijdk_test compile
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
fi
- for plugin in ${PLUGINS}; do
- verify_patchdir_still_exists
- if declare -f ${plugin}_tests >/dev/null 2>&1; then
- yetus_debug "Running ${plugin}_tests"
- #shellcheck disable=SC2086
- ${plugin}_tests
- (( RESULT = RESULT + $? ))
+ for jdkindex in ${JDK_DIR_LIST}; do
+ if [[ ${multijdkmode} == true ]]; then
+ JAVA_HOME=${jdkindex}
fi
+
+ personality_modules "${codebase}" compile
+ "${BUILDTOOL}_modules_worker" "${codebase}" compile
+ modules_messages "${codebase}" compile true
+
+ for plugin in ${PLUGINS}; do
+ verify_patchdir_still_exists
+ if declare -f ${plugin}_compile >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_compile ${codebase} ${multijdkmode}"
+ "${plugin}_compile" "${codebase}" "${multijdkmode}"
+ ((result = result + $?))
+ fi
+ done
+
done
+ JAVA_HOME=${savejavahome}
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
}
-## @description Import content from test-patch.d and optionally
-## @description from user provided plugin directory
-## @audience private
+## @description Execute the static analysis test cycle.
+## @description This will callout to _precompile, compile, and _postcompile
+## @audience public
## @stability evolving
## @replaceable no
-function importplugins
+## @param branch|patch
+## @return 0 on success
+## @return 1 on failure
+function compile_cycle
{
- local i
- local files=()
+ declare codebase=$1
+ declare result=0
+ declare plugin
- if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
- if [[ -d "${BINDIR}/test-patch.d" ]]; then
- files=(${BINDIR}/test-patch.d/*.sh)
+ find_changed_modules
+
+ for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${PLUGINS} ${TESTFORMATS}; do
+ if declare -f ${plugin}_precompile >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_precompile"
+ #shellcheck disable=SC2086
+ ${plugin}_precompile ${codebase}
+ if [[ $? -gt 0 ]]; then
+ ((result = result+1))
+ fi
fi
- fi
+ done
- if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
- yetus_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
- files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
- fi
+ compile "${codebase}"
- for i in "${files[@]}"; do
- if [[ -f ${i} ]]; then
- yetus_debug "Importing ${i}"
- . "${i}"
+ for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${PLUGINS} ${TESTFORMATS}; do
+ if declare -f ${plugin}_postcompile >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_postcompile"
+ #shellcheck disable=SC2086
+ ${plugin}_postcompile ${codebase}
+ if [[ $? -gt 0 ]]; then
+ ((result = result+1))
+ fi
fi
done
- if [[ -z ${PERSONALITY}
- && -f "${BINDIR}/personality/${PROJECT_NAME}.sh" ]]; then
- PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
- fi
-
- if [[ -n ${PERSONALITY} ]]; then
- if [[ ! -f ${PERSONALITY} ]]; then
- if [[ -f "${BINDIR}/personality/${PROJECT_NAME}.sh" ]]; then
- PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
- else
- yetus_debug "Can't find ${PERSONALITY} to import."
- return
+ for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${PLUGINS} ${TESTFORMATS}; do
+ if declare -f ${plugin}_rebuild >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_rebuild"
+ #shellcheck disable=SC2086
+ ${plugin}_rebuild ${codebase}
+ if [[ $? -gt 0 ]]; then
+ ((result = result+1))
fi
fi
- yetus_debug "Importing ${PERSONALITY}"
- . "${PERSONALITY}"
+ done
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
fi
+ return 0
}
-## @description Let plugins also get a copy of the arguments
-## @audience private
+## @description Execute the patch file test phase. Calls out to
+## @description to _patchfile
+## @audience public
## @stability evolving
## @replaceable no
-function parse_args_plugins
+## @param branch|patch
+## @return 0 on success
+## @return 1 on failure
+function patchfiletests
{
- for plugin in ${PLUGINS} ${BUGSYSTEMS} ${TESTFORMATS}; do
- if declare -f ${plugin}_parse_args >/dev/null 2>&1; then
- yetus_debug "Running ${plugin}_parse_args"
+ declare plugin
+ declare result=0
+
+ for plugin in ${BUILDTOOL} ${PLUGINS} ${TESTFORMATS}; do
+ if declare -f ${plugin}_patchfile >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_patchfile"
#shellcheck disable=SC2086
- ${plugin}_parse_args "$@"
- (( RESULT = RESULT + $? ))
+ ${plugin}_patchfile "${PATCH_DIR}/patch"
+ if [[ $? -gt 0 ]]; then
+ ((result = result+1))
+ fi
fi
done
- BUGCOMMENTS=${BUGCOMMENTS:-${BUGSYSTEMS}}
- if [[ ! ${BUGCOMMENTS} =~ console ]]; then
- BUGCOMMENTS="${BUGCOMMENTS} console"
+ if [[ ${result} -gt 0 ]]; then
+ return 1
fi
-
- BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}}
+ return 0
}
-## @description Register test-patch.d plugins
-## @audience public
-## @stability stable
-## @replaceable no
-function add_plugin
-{
- PLUGINS="${PLUGINS} $1"
-}
-## @description Register test-patch.d bugsystems
+## @description Wipe the repo clean to not invalidate tests
## @audience public
-## @stability stable
+## @stability evolving
## @replaceable no
-function add_bugsystem
+## @return 0 on success
+## @return 1 on failure
+function distclean
{
- BUGSYSTEMS="${BUGSYSTEMS} $1"
-}
+ declare result=0
+ declare plugin
-## @description Register test-patch.d test output formats
-## @audience public
-## @stability stable
-## @replaceable no
-function add_test_format
-{
- TESTFORMATS="${TESTFORMATS} $1"
+ personality_modules branch distclean
+
+ for plugin in ${PLUGINS} ${TESTFORMATS}; do
+ if declare -f ${plugin}_clean >/dev/null 2>&1; then
+ yetus_debug "Running ${plugin}_distclean"
+ #shellcheck disable=SC2086
+ ${plugin}_clean
+ if [[ $? -gt 0 ]]; then
+ ((result = result+1))
+ fi
+ fi
+ done
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
}
-## @description Calculate the differences between the specified files
-## @description and output it to stdout.
+## @description Setup to execute
## @audience public
## @stability evolving
## @replaceable no
-function calcdiffs
+## @param $@
+## @return 0 on success
+## @return 1 on failure
+function initialize
{
- local orig=$1
- local new=$2
- local tmp=${PATCH_DIR}/pl.$$.${RANDOM}
- local count=0
- local j
+ setup_defaults
- # first, pull out just the errors
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${orig}" > "${tmp}.branch"
-
- # shellcheck disable=SC2016
- ${AWK} -F: '{print $NF}' "${new}" > "${tmp}.patch"
+ parse_args "$@"
- # compare the errors, generating a string of line
- # numbers. Sorry portability: GNU diff makes this too easy
- ${DIFF} --unchanged-line-format="" \
- --old-line-format="" \
- --new-line-format="%dn " \
- "${tmp}.branch" \
- "${tmp}.patch" > "${tmp}.lined"
+ importplugins
- # now, pull out those lines of the raw output
- # shellcheck disable=SC2013
- for j in $(cat "${tmp}.lined"); do
- # shellcheck disable=SC2086
- head -${j} "${new}" | tail -1
- done
+ parse_args_plugins "$@"
- rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
-}
+ plugins_initialize
-###############################################################################
-###############################################################################
-###############################################################################
+ finish_docker_stats
-big_console_header "Bootstrapping test harness"
+ locate_patch
-setup_defaults
+ # from here on out, we'll be in ${BASEDIR} for cwd
+ # plugins need to pushd/popd if they change.
+ git_checkout
+ RESULT=$?
+ if [[ ${JENKINS} == "true" ]] ; then
+ if [[ ${RESULT} != 0 ]] ; then
+ exit 1
+ fi
+ fi
-parse_args "$@"
+ find_changed_files
-importplugins
+ check_reexec
-parse_args_plugins "$@"
+ determine_needed_tests
-finish_docker_stats
+ prepopulate_footer
+}
-locate_patch
+## @description perform prechecks
+## @audience private
+## @stability evolving
+## @return exits on failure
+function prechecks
+{
+ declare plugin
+ declare result=0
-# from here on out, we'll be in ${BASEDIR} for cwd
-# routines need to pushd/popd if they change.
-git_checkout
-RESULT=$?
-if [[ ${JENKINS} == "true" ]] ; then
- if [[ ${RESULT} != 0 ]] ; then
- exit 100
+ verify_patch_file
+ (( result = result + $? ))
+ if [[ ${result} != 0 ]] ; then
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
fi
-fi
-find_changed_files
+ for plugin in ${BUILDTOOL} ${PLUGINS} ${TESTFORMATS}; do
+ verify_patchdir_still_exists
-check_reexec
+ if declare -f ${plugin}_precheck >/dev/null 2>&1; then
-determine_needed_tests
+ yetus_debug "Running ${plugin}_precheck"
+ #shellcheck disable=SC2086
+ ${plugin}_precheck
-postcheckout
+ (( result = result + $? ))
+ if [[ ${result} != 0 ]] ; then
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
+ fi
+ fi
+ done
+}
-fullyqualifyjdks
+###############################################################################
+###############################################################################
+###############################################################################
-prepopulate_footer
+initialize "$@"
-find_changed_modules
+prechecks
-preapply
+patchfiletests
+((RESULT=RESULT+$?))
-apply_patch_file
+compile_cycle branch
+((RESULT=RESULT+$?))
-# we find changed modules again
-# in case the patch adds or removes a module
-# this also means that test suites need to be
-# aware that there might not be a 'before'
-find_changed_modules
+distclean
-postapply
+apply_patch_file
-check_mvninstall
+compute_gitdiff
-postinstall
+compile_cycle patch
+((RESULT=RESULT+$?))
runtests
+((RESULT=RESULT+$?))
finish_vote_table
[19/50] [abbrv] yetus git commit: HADOOP-12275. releasedocmaker:
unreleased should still be dated (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12275. releasedocmaker: unreleased should still be dated (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/b44bde58
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/b44bde58
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/b44bde58
Branch: refs/heads/master
Commit: b44bde5849f9a1ee784bb6eddebc2e9a45ed3426
Parents: ff2115c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 10 09:44:47 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 10 09:44:47 2015 -0700
----------------------------------------------------------------------
dev-support/releasedocmaker.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/b44bde58/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 37bd58a..3c398be 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -434,7 +434,7 @@ def main():
elif options.usetoday:
reldate = strftime("%Y-%m-%d", gmtime())
else:
- reldate = "Unreleased"
+ reldate = "Unreleased (as of %s)" % strftime("%Y-%m-%d", gmtime())
if not os.path.exists(vstr):
os.mkdir(vstr)
[48/50] [abbrv] yetus git commit: HADOOP-12397. Incomplete comment
for test-patch compile_cycle function (Jagadesh Kiran N via aw)
Posted by bu...@apache.org.
HADOOP-12397. Incomplete comment for test-patch compile_cycle function (Jagadesh Kiran N via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/61780962
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/61780962
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/61780962
Branch: refs/heads/master
Commit: 61780962d86d415d399871d4221938937a00afe1
Parents: 0222626
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 11 17:56:27 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 11 17:56:27 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/61780962/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index f311076..eb25aea 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -2779,7 +2779,7 @@ function compile
}
## @description Execute the static analysis test cycle.
-## @description This will callout to _precompile, compile, and _postcompile
+## @description This will callout to _precompile, compile, _postcompile and _rebuild
## @audience public
## @stability evolving
## @replaceable no
[35/50] [abbrv] yetus git commit: HADOOP-12233. if CHANGED_FILES is
corrupt, find_changed_modules never returns (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12233. if CHANGED_FILES is corrupt, find_changed_modules never returns (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/90b8e664
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/90b8e664
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/90b8e664
Branch: refs/heads/master
Commit: 90b8e664f0755c540629135976e8dea079b47ade
Parents: 2340c55
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 25 09:15:22 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Aug 25 09:15:22 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/90b8e664/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 4d4b63f..9f08ad3 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1107,6 +1107,7 @@ function find_buildfile_dir
{
local buildfile=$1
local dir=$2
+ local d
yetus_debug "Find ${buildfile} dir for: ${dir}"
@@ -1119,6 +1120,12 @@ function find_buildfile_dir
yetus_debug "ERROR: ${buildfile} is not found."
return 1
else
+ d=$(cd -P -- "$(dirname -- "${dir}")" >/dev/null && pwd -P)
+ relative_dir "${d}" >/dev/null
+ if [[ $? == 1 ]]; then
+ yetus_debug "ERROR: ${dir} is not in ${BASEDIR}."
+ return 1
+ fi
dir=$(dirname "${dir}")
fi
done
@@ -1151,6 +1158,7 @@ function find_changed_files
function module_skipdir
{
local dir=${1}
+ local d
local i
yetus_debug "Checking skipdirs for ${dir}"
@@ -1170,6 +1178,12 @@ function module_skipdir
if [[ ${dir} == "." ]]; then
return 0
else
+ d=$(cd -P -- "$(dirname -- "${dir}")" >/dev/null && pwd -P)
+ relative_dir "${d}" >/dev/null
+ if [[ $? == 1 ]]; then
+ yetus_debug "ERROR: ${dir} is not in ${BASEDIR}."
+ return 1
+ fi
dir=$(dirname "${dir}")
yetus_debug "Trying to skip: ${dir}"
fi
[31/50] [abbrv] yetus git commit: HADOOP-12129. rework test-patch bug
system support (aw)
Posted by bu...@apache.org.
HADOOP-12129. rework test-patch bug system support (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/463891f3
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/463891f3
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/463891f3
Branch: refs/heads/master
Commit: 463891f3fbedb0e3b32749e577dc08dcf29a8152
Parents: 0d65ccd
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 17 08:10:35 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 17 08:10:35 2015 -0700
----------------------------------------------------------------------
dev-support/docs/precommit-advanced.md | 59 +-
dev-support/docs/precommit-basic.md | 50 +-
dev-support/personality/flink.sh | 4 +-
dev-support/personality/hadoop.sh | 4 +-
dev-support/personality/hbase.sh | 4 +-
dev-support/personality/pig.sh | 4 +-
dev-support/personality/tajo.sh | 4 +-
dev-support/personality/tez.sh | 4 +-
dev-support/smart-apply-patch.sh | 22 +-
.../test-patch-docker/Dockerfile-startstub | 15 +-
dev-support/test-patch.d/builtin-bugsystem.sh | 163 ++++++
dev-support/test-patch.d/github.sh | 411 +++++++++++++-
dev-support/test-patch.d/jira.sh | 305 +++++++++--
dev-support/test-patch.d/shellcheck.sh | 6 +-
dev-support/test-patch.d/whitespace.sh | 23 +
dev-support/test-patch.sh | 533 +++++++------------
16 files changed, 1174 insertions(+), 437 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/docs/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-advanced.md b/dev-support/docs/precommit-advanced.md
index 3185512..7830afe 100644
--- a/dev-support/docs/precommit-advanced.md
+++ b/dev-support/docs/precommit-advanced.md
@@ -52,8 +52,9 @@ test-patch always passes -noinput to Ant. This force ant to be non-interactive.
test-patch allows one to add to its basic feature set via plug-ins. There is a directory called test-patch.d off of the directory where test-patch.sh lives. Inside this directory one may place some bash shell fragments that, if setup with proper functions, will allow for test-patch to call it as necessary.
+## Test Plug-ins
-Every plugin must have one line in order to be recognized:
+Every test plugin must have one line in order to be recognized:
```bash
add_plugin <pluginname>
@@ -69,33 +70,69 @@ This function gets called for every file that a patch may contain. This allows
Similarly, there are other functions that may be defined during the test-patch run:
-* pluginname_postcheckout
+* pluginname\_postcheckout
- executed prior to the patch being applied but after the git repository is setup. This is useful for any early error checking that might need to be done before any heavier work.
-* pluginname_preapply
+* pluginname\_preapply
- executed prior to the patch being applied. This is useful for any "before"-type data collection for later comparisons.
-* pluginname_postapply
+* pluginname\_postapply
- executed after the patch has been applied. This is useful for any "after"-type data collection.
-* pluginname_postinstall
+* pluginname\_postinstall
- executed after the mvn install test has been done. If any tests require the Maven repository to be up-to-date with the contents of the patch, this is the place.
-* pluginname_tests
+* pluginname\_tests
- executed after the unit tests have completed.
If the plug-in has some specific options, one can use following functions:
-* pluginname_usage
+* pluginname\_usage
- executed when the help message is displayed. This is used to display the plug-in specific options for the user.
-* pluginname_parse_args
+* pluginname\_parse\_args
- executed prior to any other above functions except for pluginname_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
HINT: It is recommended to make the pluginname relatively small, 10 characters at the most. Otherwise, the ASCII output table may be skewed.
+## Bug System Plug-ins
+
+Similar to tests, the ability to add support for bug tracking systems is also handled via a plug-in mechanism.
+
+* pluginname_usage
+
+ - executed when the help message is displayed. This is used to display the plug-in specific options for the user.
+
+* pluginname\_parse\_args
+
+ - executed prior to any other above functions except for pluginname_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
+
+
+* pluginname\_locate\_patch
+
+ - Given input from the user, download the patch if possible.
+
+* pluginname\_determine\_branch
+
+ - Using any heuristics available, return the branch to process, if possible.
+
+* pluginname\_determine\_issue
+
+ - Using any heuristics available, set the issue, bug number, etc, for this bug system, if possible. This is typically used to fill in supplementary information in the final output table.
+
+* pluginname\_writecomment
+
+ - Given text input, write this output to the bug system as a comment. NOTE: It is the bug system's responsibility to format appropriately.
+
+* pluginname\_linecomments
+
+ - This function allows for the system to write specific comments on specific lines if the bug system supports code review comments.
+
+* pluginname\_finalreport
+
+ - Write the final result table to the bug system.
# Configuring for Other Projects
@@ -181,7 +218,7 @@ This function will tell test-patch that when the javadoc test is being run, do t
# Important Variables
-There are a handful of extremely important variables that make life easier for personality and plug-in writers:
+There are a handful of extremely important system variables that make life easier for personality and plug-in writers. Other variables may be provided by individual plug-ins. Check their development documentation for more information.
* BUILD\_NATIVE will be set to true if the system has requested that non-JVM-based code be built (e.g., JNI or other compiled C code). Under Jenkins, this is always true.
@@ -193,9 +230,11 @@ There are a handful of extremely important variables that make life easier for p
* CHANGED\_MODULES reports which modules that appear to have source code in them.
+* GITHUB\_REPO is to help test-patch when talking to Github. If test-patch is given just a number on the command line, it will default to using this repo to determine the pull request.
+
* HOW\_TO\_CONTRIBUTE should be a URL that points to a project's on-boarding documentation for new users. Currently, it is used to suggest a review of patch naming guidelines. Since this should be project specific information, it is useful to set in a project's personality.
-* ISSUE\_RE is to help test-patch when talking to JIRA. It helps determine if the given project is appropriate for the given JIRA issue.
+* JIRA\_ISSUE\_RE is to help test-patch when talking to JIRA. It helps determine if the given project is appropriate for the given JIRA issue.
* MODULE and other MODULE\_\* are arrays that contain which modules, the status, etc, to be operated upon. These should be treated as read-only by plug-ins.
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/docs/precommit-basic.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-basic.md b/dev-support/docs/precommit-basic.md
index e68ad07..a612214 100644
--- a/dev-support/docs/precommit-basic.md
+++ b/dev-support/docs/precommit-basic.md
@@ -48,9 +48,9 @@ test-patch has the following requirements:
* POSIX awk
* POSIX grep
* POSIX sed
-* wget
+* curl
* file command
-* smart-apply-patch.sh
+* smart-apply-patch.sh (included!)
Maven plugins requirements:
@@ -59,8 +59,8 @@ Maven plugins requirements:
Optional:
-* Apache JIRA-based issue tracking
-* JIRA cli tools
+* JIRA-based issue tracking
+* GitHub-based issue tracking
The locations of these files are (mostly) assumed to be in the file path, but may be overridden via command line options. For Solaris and Solaris-like operating systems, the default location for the POSIX binaries is in /usr/xpg4/bin and the default location for the GNU binaries is /usr/gnu/bin.
@@ -119,6 +119,8 @@ will tell test-patch to use ant instead of maven to drive the project.
# Providing Patch Files
+## JIRA
+
It is a fairly common practice within the Apache community to use Apache's JIRA instance to store potential patches. As a result, test-patch supports providing just a JIRA issue number. test-patch will find the *last* attachment, download it, then process it.
For example:
@@ -129,15 +131,47 @@ $ test-patch.sh (other options) HADOOP-9905
... will process the patch file associated with this JIRA issue.
-A new practice is to use a service such as GitHub and its Pull Request (PR) feature. Luckily, test-patch supports URLs and many services like GitHub provide ways to provide unified diffs via URLs.
+If the Apache JIRA system is not in use, then override options may be provided on the command line to point to a different JIRA instance.
+
+```bash
+$ test-patch.sh --jira-issue-re='^PROJECT-[0-9]+$' --jira-base-url='https://example.com/jira' PROJECT-90
+```
+
+... will process the patch file attached to PROJECT-90 on the JIRA instance located on the example.com server.
+
+## GITHUB
+
+test-patch has some basic support for Github. test-patch supports many forms of providing pull requests to work on:
+
+```bash
+$ test-patch.sh --github-repo=apache/pig 99
+```
+
+or
+
+```bash
+$ test-patch.sh https://github.com/apache/pig/pulls/99
+```
+
+or
+
+```bash
+$ test-patch.sh https://github.com/apache/pig/pulls/99.patch
+```
+
+... will process PR #99 on the apache/pig repo.
+
+## Generic URLs
+
+Luckily, test-patch supports provide ways to provide unified diffs via URLs.
For example:
```bash
-$ test-patch.sh (other options) https://github.com/apache/flink/pull/773.patch
+$ test-patch.sh (other options) https://example.com/webserver/file.patch
```
-... will grab a unified diff of PR #773 and process it.
+... will download and process the file.patch from the example.com webserver.
# Project-specific Capabilities
@@ -181,8 +215,6 @@ $ test-patch.sh (other options) --docker
This will do some preliminary setup and then re-execute itself inside a Docker container. For more information on how to provide a custom Dockerfile, see the advanced guide.
-
-
## In Closing
test-patch has many other features and command line options for the basic user. Many of these are self-explanatory. To see the list of options, run test-patch.sh without any options or with --help.
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/personality/flink.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/flink.sh b/dev-support/personality/flink.sh
index de2a0f1..4b6c390 100755
--- a/dev-support/personality/flink.sh
+++ b/dev-support/personality/flink.sh
@@ -17,7 +17,9 @@
#shellcheck disable=SC2034
PATCH_BRANCH_DEFAULT=master
#shellcheck disable=SC2034
-ISSUE_RE='^FLINK-[0-9]+$'
+JIRA_ISSUE_RE='^FLINK-[0-9]+$'
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/flink"
#shellcheck disable=SC2034
HOW_TO_CONTRIBUTE=""
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hadoop.sh b/dev-support/personality/hadoop.sh
index 1243a17..b3eb04a 100755
--- a/dev-support/personality/hadoop.sh
+++ b/dev-support/personality/hadoop.sh
@@ -21,7 +21,9 @@ PATCH_BRANCH_DEFAULT=trunk
#shellcheck disable=SC2034
HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
#shellcheck disable=SC2034
-ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
+JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/hadoop"
#shellcheck disable=SC2034
PYLINT_OPTIONS="--indent-string=' '"
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hbase.sh b/dev-support/personality/hbase.sh
index 9749096..4f23679 100755
--- a/dev-support/personality/hbase.sh
+++ b/dev-support/personality/hbase.sh
@@ -17,7 +17,9 @@
#shellcheck disable=SC2034
PATCH_BRANCH_DEFAULT=master
#shellcheck disable=SC2034
-ISSUE_RE='^HBASE-[0-9]+$'
+JIRA_ISSUE_RE='^HBASE-[0-9]+$'
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/hbase"
#shellcheck disable=SC2034
HOW_TO_CONTRIBUTE=""
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/personality/pig.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/pig.sh b/dev-support/personality/pig.sh
index d01a410..d67b227 100755
--- a/dev-support/personality/pig.sh
+++ b/dev-support/personality/pig.sh
@@ -17,7 +17,9 @@
#shellcheck disable=SC2034
PATCH_BRANCH_DEFAULT=trunk
#shellcheck disable=SC2034
-ISSUE_RE='^PIG-[0-9]+$'
+JIRA_ISSUE_RE='^PIG-[0-9]+$'
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/pig"
#shellcheck disable=SC2034
HOW_TO_CONTRIBUTE=""
#shellcheck disable=SC2034
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/personality/tajo.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/tajo.sh b/dev-support/personality/tajo.sh
index 56e5442..7e7ea97 100755
--- a/dev-support/personality/tajo.sh
+++ b/dev-support/personality/tajo.sh
@@ -17,7 +17,9 @@
#shellcheck disable=SC2034
PATCH_BRANCH_DEFAULT=master
#shellcheck disable=SC2034
-ISSUE_RE='^TAJO-[0-9]+$'
+JIRA_ISSUE_RE='^TAJO-[0-9]+$'
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/tajo"
#shellcheck disable=SC2034
HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/personality/tez.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/tez.sh b/dev-support/personality/tez.sh
index 1d6a227..9b45759 100755
--- a/dev-support/personality/tez.sh
+++ b/dev-support/personality/tez.sh
@@ -17,7 +17,9 @@
#shellcheck disable=SC2034
PATCH_BRANCH_DEFAULT=master
#shellcheck disable=SC2034
-ISSUE_RE='^TEZ-[0-9]+$'
+JIRA_ISSUE_RE='^TEZ-[0-9]+$'
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/tez"
#shellcheck disable=SC2034
HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/smart-apply-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
index 00e3a0a..e11a734 100755
--- a/dev-support/smart-apply-patch.sh
+++ b/dev-support/smart-apply-patch.sh
@@ -76,7 +76,7 @@ function setup_defaults
SunOS)
AWK=${AWK:-/usr/xpg4/bin/awk}
SED=${SED:-/usr/xpg4/bin/sed}
- WGET=${WGET:-wget}
+ CURL=${CURL:-curl}
GIT=${GIT:-git}
GREP=${GREP:-/usr/xpg4/bin/grep}
PATCH=${PATCH:-/usr/gnu/bin/patch}
@@ -86,7 +86,7 @@ function setup_defaults
*)
AWK=${AWK:-awk}
SED=${SED:-sed}
- WGET=${WGET:-wget}
+ CURL=${CURL:-curl}
GIT=${GIT:-git}
GREP=${GREP:-grep}
PATCH=${PATCH:-patch}
@@ -122,7 +122,7 @@ function yetus_usage
echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
echo "--patch-cmd=<cmd> The GNU-compatible 'patch' command to use (default 'patch')"
- echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
+ echo "--curl-cmd=<cmd> The 'curl' command to use (default 'curl')"
}
## @description Interpret the command line parameters
@@ -162,8 +162,8 @@ function parse_args
--patch-dir=*)
PATCH_DIR=${i#*=}
;;
- --wget-cmd=*)
- WGET=${i#*=}
+ --curl-cmd=*)
+ CURL=${i#*=}
;;
--*)
## PATCH_OR_ISSUE can't be a --. So this is probably
@@ -233,13 +233,15 @@ function locate_patch
echo "Patch is being downloaded at $(date) from"
PATCHURL="${PATCH_OR_ISSUE}"
else
- ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
-
+ ${CURL} --silent \
+ --output "${PATCH_DIR}/jira" \
+ --location \
+ "https://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
case $? in
0)
;;
2)
- yetus_error "ERROR: .wgetrc/.netrc parsing error."
+ yetus_error "ERROR: .curlrc/.netrc parsing error."
cleanup_and_exit 1
;;
3)
@@ -269,7 +271,7 @@ function locate_patch
#shellcheck disable=SC2016
relativePatchURL=$(${AWK} 'match($0,"\"/jira/secure/attachment/[0-9]*/[^\"]*"){print substr($0,RSTART+1,RLENGTH-1)}' "${PATCH_DIR}/jira" |
${GREP} -v -e 'htm[l]*$' | sort | tail -1)
- PATCHURL="http://issues.apache.org${relativePatchURL}"
+ PATCHURL="https://issues.apache.org${relativePatchURL}"
if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
notSureIfPatch=true
fi
@@ -277,7 +279,7 @@ function locate_patch
fi
fi
if [[ -z "${PATCH_FILE}" ]]; then
- ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
+ ${CURL} --silent --location --output "${PATCH_DIR}/patch" "${PATCHURL}"
if [[ $? != 0 ]];then
yetus_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
cleanup_and_exit 1
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/Dockerfile-startstub b/dev-support/test-patch-docker/Dockerfile-startstub
index fd3e4c5..c49b589 100644
--- a/dev-support/test-patch-docker/Dockerfile-startstub
+++ b/dev-support/test-patch-docker/Dockerfile-startstub
@@ -62,8 +62,8 @@ RUN apt-get install -y oracle-java8-installer
# Install findbugs
######
RUN mkdir -p /opt/findbugs && \
- wget https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
- -O /opt/findbugs.tar.gz && \
+ curl https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
+ -o /opt/findbugs.tar.gz && \
tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
ENV FINDBUGS_HOME /opt/findbugs
@@ -83,14 +83,3 @@ RUN gem install rubocop
###
RUN gem install ruby-lint
-#####
-# Install JIRA CLI
-#####
-
-RUN mkdir -p /opt/jiracli && \
- wget https://bobswift.atlassian.net/wiki/download/attachments/16285777/jira-cli-2.2.0-distribution.zip \
- -O /tmp/jiracli.zip && \
- unzip -qq -d /opt/jiracli /tmp/jiracli.zip && \
- ln -s /opt/jiracli/jira-cli-2.2.0 /opt/jiracli/latest && \
- chmod -R a+rx /opt/jiracli/jira-cli-2.2.0
-ENV JIRACLI_HOME /opt/jiracli/latest
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/test-patch.d/builtin-bugsystem.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/builtin-bugsystem.sh b/dev-support/test-patch.d/builtin-bugsystem.sh
new file mode 100644
index 0000000..9a9ee05
--- /dev/null
+++ b/dev-support/test-patch.d/builtin-bugsystem.sh
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This bug system handles the output on the screen.
+
+add_bugsystem console
+
+# we always call this one last
+
+function generic_locate_patch
+{
+ declare input=$1
+ declare output=$2
+
+ if [[ "${OFFLINE}" == true ]]; then
+ yetus_debug "generic_locate_patch: offline, skipping"
+ return 1
+ fi
+
+ ${CURL} --silent \
+ --output "${output}" \
+ "${input}"
+ if [[ $? != 0 ]]; then
+ yetus_debug "jira_locate_patch: not a JIRA."
+ return 1
+ fi
+ return 0
+}
+
+## @description Print out the finished details on the console
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param runresult
+## @return 0 on success
+## @return 1 on failure
+function console_finalreport
+{
+ declare result=$1
+ shift
+ declare i=0
+ declare ourstring
+ declare vote
+ declare subs
+ declare ela
+ declare comment
+ declare commentfile1="${PATCH_DIR}/comment.1"
+ declare commentfile2="${PATCH_DIR}/comment.2"
+ declare normaltop
+ declare line
+ declare seccoladj=0
+ declare spcfx=${PATCH_DIR}/spcl.txt
+
+ if [[ ${result} == 0 ]]; then
+ if [[ ${JENKINS} == false ]]; then
+ {
+ printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
+ printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
+ printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
+ printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
+ printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
+ printf "ICB8bXwgICB8bXwgIAo"
+ } > "${spcfx}"
+ fi
+ printf "\n\n+1 overall\n\n"
+ else
+ if [[ ${JENKINS} == false ]]; then
+ {
+ printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
+ printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
+ printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
+ printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
+ printf "ICAgICAgICAgICAgICAgICAK"
+ } > "${spcfx}"
+ fi
+ printf "\n\n-1 overall\n\n"
+ fi
+
+ if [[ -f ${spcfx} ]]; then
+ if which base64 >/dev/null 2>&1; then
+ base64 --decode "${spcfx}" 2>/dev/null
+ elif which openssl >/dev/null 2>&1; then
+ openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
+ fi
+ echo
+ echo
+ rm "${spcfx}"
+ fi
+
+ seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}")
+ if [[ ${seccoladj} -lt 10 ]]; then
+ seccoladj=10
+ fi
+
+ seccoladj=$((seccoladj + 2 ))
+ i=0
+ until [[ $i -eq ${#TP_HEADER[@]} ]]; do
+ printf "%s\n" "${TP_HEADER[${i}]}"
+ ((i=i+1))
+ done
+
+ printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
+ echo "============================================================================"
+ i=0
+ until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
+ ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ')
+ vote=$(echo "${ourstring}" | cut -f2 -d\|)
+ subs=$(echo "${ourstring}" | cut -f3 -d\|)
+ ela=$(echo "${ourstring}" | cut -f4 -d\|)
+ comment=$(echo "${ourstring}" | cut -f5 -d\|)
+
+ echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
+ normaltop=$(head -1 "${commentfile1}")
+ ${SED} -e '1d' "${commentfile1}" > "${commentfile2}"
+
+ printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
+ "${subs}" "${ela}" "${normaltop}"
+ while read line; do
+ printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}"
+ done < "${commentfile2}"
+
+ ((i=i+1))
+ rm "${commentfile2}" "${commentfile1}" 2>/dev/null
+ done
+
+ if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
+ seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}")
+ printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
+ i=0
+ until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do
+ ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ')
+ vote=$(echo "${ourstring}" | cut -f2 -d\|)
+ subs=$(echo "${ourstring}" | cut -f3 -d\|)
+ printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
+ ((i=i+1))
+ done
+ fi
+
+ printf "\n\n|| Subsystem || Report/Notes ||\n"
+ echo "============================================================================"
+ i=0
+
+ until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
+ comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
+ ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
+ printf "%s\n" "${comment}"
+ ((i=i+1))
+ done
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/test-patch.d/github.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/github.sh b/dev-support/test-patch.d/github.sh
index 281f15b..36c7e51 100755
--- a/dev-support/test-patch.d/github.sh
+++ b/dev-support/test-patch.d/github.sh
@@ -14,23 +14,403 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# This bug system provides github integration
+
add_bugsystem github
+# personalities can override the following settings:
+
+# Web interface URL.
+GITHUB_BASE_URL="https://github.com"
+
+# API interface URL.
+GITHUB_API_URL="https://api.github.com"
+
+# user/repo
+GITHUB_REPO=""
+
+# user settings
+GITHUB_PASSWD=""
+GITHUB_TOKEN=""
+GITHUB_USER=""
+GITHUB_ISSUE=""
+
+# private globals...
+GITHUB_BRIDGED=false
+GITHUB_COMMITSHA=""
+
+function github_usage
+{
+ echo "GITHUB Options:"
+ echo "--github-api-url=<url> The URL of the API for github (default: '${GITHUB_API_URL}')"
+ echo "--github-base-url=<url> The URL of the github server (default:'${GITHUB_BASE_URL}')"
+ echo "--github-password=<pw> Github password"
+ echo "--github-repo=<repo> github repo to use (default:'${GITHUB_REPO}')"
+ echo "--github-token=<token> The token to use to write to github"
+ echo "--github-user=<user> Github user"
+}
+
+function github_parse_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --github-api-url=*)
+ GITHUB_API_URL=${i#*=}
+ ;;
+ --github-base-url=*)
+ GITHUB_BASE_URL=${i#*=}
+ ;;
+ --github-repo=*)
+ GITHUB_REPO=${i#*=}
+ ;;
+ --github-token=*)
+ GITHUB_TOKEN=${i#*=}
+ ;;
+ --github-password=*)
+ GITHUB_PASSWD=${i#*=}
+ ;;
+ --github-user=*)
+ GITHUB_USER=${i#*=}
+ ;;
+ esac
+ done
+}
+
+## @description this gets called when JIRA thinks this
+## @description issue is just a pointer to github
+## @description WARNING: Called from JIRA plugin!
+function github_jira_bridge
+{
+ declare fileloc=$1
+ declare urlfromjira
+
+ # we use this to prevent loops later on
+ GITHUB_BRIDGED=true
+
+ # the JIRA issue has already been downloaded. So let's
+ # find the URL. This is currently hard-coded to github.com
+ # Sorry Github Enterprise users. :(
+
+ # shellcheck disable=SC2016
+ urlfromjira=$(${AWK} 'match($0,"https://github.com/.*patch"){print $1}' "${PATCH_DIR}/jira" | tail -1)
+ github_breakup_url "${urlfromjira}"
+ github_locate_patch "${GITHUB_ISSUE}" "${fileloc}"
+}
+
+## @description given a URL, break it up into github plugin globals
+## @description this will *override* any personality or yetus defaults
+## @params url
+function github_breakup_url
+{
+ declare url=$1
+ declare count
+ declare pos1
+ declare pos2
+
+ count=${url//[^\/]}
+ count=${#count}
+ ((pos2=count-3))
+ ((pos1=pos2))
+
+ GITHUB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/)
+
+ ((pos1=pos1+1))
+ ((pos2=pos1+1))
+
+ GITHUB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/)
+
+ ((pos1=pos2+2))
+ unset pos2
+
+ GITHUB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.)
+}
+
+
+## @description based upon a github PR, attempt to link back to JIRA
+function github_find_jira_title
+{
+ declare title
+ declare maybe
+ declare retval
+
+ if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then
+ return 1
+ fi
+
+ title=$(GREP title "${PATCH_DIR}/github-pull.json" \
+ | cut -f4 -d\")
+
+ # people typically do two types: JIRA-ISSUE: and [JIRA-ISSUE]
+ # JIRA_ISSUE_RE is pretty strict so we need to chop that stuff
+ # out first
+
+ maybe=$(echo "${title}" | cut -f2 -d\[ | cut -f1 -d\])
+ jira_determine_issue "${maybe}"
+ retval=$?
+
+ if [[ ${retval} == 0 ]]; then
+ return 0
+ fi
+
+ maybe=$(echo "${title}" | cut -f1 -d:)
+ jira_determine_issue "${maybe}"
+ retval=$?
+
+ if [[ ${retval} == 0 ]]; then
+ return 0
+ fi
+}
+
+function github_determine_issue
+{
+ declare input=$1
+
+ if [[ ${input} =~ ^[0-9]+$
+ && -n ${GITHUB_REPO} ]]; then
+ # shellcheck disable=SC2034
+ ISSUE=${input}
+ if [[ -z ${GITHUB_ISSUE} ]]; then
+ GITHUB_ISSUE=${input}
+ fi
+ fi
+
+ # if JIRA didn't call us, should we call it?
+ if [[ ${GITHUB_BRIDGED} == false ]]; then
+ github_find_jira_title
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+ fi
+
+ if [[ -n ${GITHUB_ISSUE} ]]; then
+ return 0
+ fi
+
+ return 1
+}
+
+## @description Try to guess the branch being tested using a variety of heuristics
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success, with PATCH_BRANCH updated appropriately
+## @return 1 on failure
+function github_determine_branch
+{
+ if [[ ! -f "${PATCH_DIR}/github-pull.json" ]]; then
+ return 1
+ fi
+
+ # shellcheck disable=SC2016
+ PATCH_BRANCH=$(${AWK} 'match($0,"\"ref\": \""){print $2}' "${PATCH_DIR}/github-pull.json"\
+ | cut -f2 -d\"\
+ | tail -1 )
+
+ yetus_debug "Github determine branch: starting with ${PATCH_BRANCH}"
+
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+ return 1
+}
+
+function github_locate_patch
+{
+ declare input=$1
+ declare output=$2
+ declare githubauth
+
+ if [[ "${OFFLINE}" == true ]]; then
+ yetus_debug "github_locate_patch: offline, skipping"
+ return 1
+ fi
+
+
+ # https://github.com/your/repo/pull/##
+ if [[ ${input} =~ ^${GITHUB_BASE_URL}.*/pull/[0-9]+$ ]]; then
+ github_breakup_url "${input}.patch"
+ input=${GITHUB_ISSUE}
+ fi
+
+ # https://github.com/your/repo/pulls/##.patch
+ if [[ ${input} =~ ^${GITHUB_BASE_URL}.*patch$ ]]; then
+ github_breakup_url "${input}"
+ input=${GITHUB_ISSUE}
+ fi
+
+ # https://github.com/your/repo/pulls/##.diff
+ if [[ ${input} =~ ^${GITHUB_BASE_URL}.*diff$ ]]; then
+ github_breakup_url "${input}"
+ input=${GITHUB_ISSUE}
+ fi
+
+ # if it isn't a number at this point, no idea
+ # how to process
+ if [[ ! ${input} =~ ^[0-9]+$ ]]; then
+ yetus_debug "github: ${input} is not a pull request #"
+ return 1
+ fi
+
+ # we always pull the .patch version (even if .diff was given)
+ # with the assumption that this way binary files work.
+ # The downside of this is that the patch files are
+ # significantly larger and therefore take longer to process
+ PATCHURL="${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}.patch"
+ echo "GITHUB PR #${input} is being downloaded at $(date) from"
+ echo "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}"
+
+ if [[ -n "${GITHUB_USER}"
+ && -n "${GITHUB_PASSWD}" ]]; then
+ githubauth="${GITHUB_USER}:${GITHUB_PASSWD}"
+ elif [[ -n "${GITHUB_TOKEN}" ]]; then
+ githubauth="Authorization: token ${GITHUB_TOKEN}"
+ else
+ githubauth="X-ignore-me: fake"
+ fi
+
+ # Let's pull the PR JSON for later use
+ ${CURL} --silent --fail \
+ -H "Accept: application/vnd.github.v3.full+json" \
+ -H "${githubauth}" \
+ --output "${PATCH_DIR}/github-pull.json" \
+ --location \
+ "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${input}"
+
+ echo "Patch from GITHUB PR #${input} is being downloaded at $(date) from"
+ echo "${PATCHURL}"
+
+ # the actual patch file
+ ${CURL} --silent --fail \
+ --output "${output}" \
+ --location \
+ -H "${githubauth}" \
+ "${PATCHURL}"
+
+ if [[ $? != 0 ]]; then
+ yetus_debug "github_locate_patch: not a github pull request."
+ return 1
+ fi
+
+ GITHUB_ISSUE=${input}
+
+ # github will translate this to be #(xx) !
+ add_footer_table "GITHUB PR" "${GITHUB_BASE_URL}/${GITHUB_REPO}/pull/${input}"
+
+ return 0
+}
+
+function github_linecomments
+{
+ declare plugin=$1
+ declare file=$2
+ # shellcheck disable=SC2034
+ declare realline=$3
+ declare uniline=$4
+ declare text=$5
+ declare tempfile="${PATCH_DIR}/ghcomment.$$.${RANDOM}"
+ declare githubauth
+
+ if [[ "${file}" =~ ^./ ]]; then
+ file=${file##./}
+ fi
+
+ if [[ -z "${GITHUB_COMMITSHA}" ]]; then
+ GITHUB_COMMITSHA=$(${GREP} \"sha\" "${PATCH_DIR}/github-pull.json" 2>/dev/null \
+ | head -1 \
+ | cut -f4 -d\")
+ fi
+
+ if [[ -z "${uniline}" ]]; then
+ return
+ fi
+
+ # build our REST post
+ {
+ printf "{\"body\":\""
+ echo "${plugin}: ${text}" \
+ | ${SED} -e 's,\\,\\\\,g' \
+ -e 's,\",\\\",g' \
+ -e 's,$,\\r\\n,g' \
+ | tr -d '\n'
+ echo "\","
+ echo "\"commit_id\":\"${GITHUB_COMMITSHA}\","
+ echo "\"path\":\"${file}\","
+ echo "\"position\":${uniline}"
+ echo "}"
+ } > "${tempfile}"
+
+ if [[ -n "${GITHUB_USER}"
+ && -n "${GITHUB_PASSWD}" ]]; then
+ githubauth="${GITHUB_USER}:${GITHUB_PASSWD}"
+ elif [[ -n "${GITHUB_TOKEN}" ]]; then
+ githubauth="Authorization: token ${GITHUB_TOKEN}"
+ else
+ return 0
+ fi
+
+ ${CURL} -X POST \
+ -H "Accept: application/vnd.github.v3.full+json" \
+ -H "Content-Type: application/json" \
+ -H "${githubauth}" \
+ -d @"${tempfile}" \
+ --silent --location \
+ "${GITHUB_API_URL}/repos/${GITHUB_REPO}/pulls/${GITHUB_ISSUE}/comments" \
+ >/dev/null
+ rm "${tempfile}"
+}
+
## @description Write the contents of a file to github
## @params filename
## @stability stable
## @audience public
function github_write_comment
{
- local -r commentfile=${1}
- shift
+ declare -r commentfile=${1}
+ declare retval=0
+ declare restfile="${PATCH_DIR}/ghcomment.$$"
+ declare githubauth
- local retval=1
+ if [[ "${OFFLINE}" == true ]]; then
+ echo "Github Plugin: Running in offline, comment skipped."
+ return 0
+ fi
+ {
+ printf "{\"body\":\""
+ ${SED} -e 's,\\,\\\\,g' \
+ -e 's,\",\\\",g' \
+ -e 's,$,\\r\\n,g' "${commentfile}" \
+ | tr -d '\n'
+ echo "\"}"
+ } > "${restfile}"
+
+ if [[ -n "${GITHUB_USER}"
+ && -n "${GITHUB_PASSWD}" ]]; then
+ githubauth="${GITHUB_USER}:${GITHUB_PASSWD}"
+ elif [[ -n "${GITHUB_TOKEN}" ]]; then
+ githubauth="Authorization: token ${GITHUB_TOKEN}"
+ else
+ echo "Github Plugin: no credentials provided to write a comment."
+ return 0
+ fi
+
+ ${CURL} -X POST \
+ -H "Accept: application/vnd.github.v3.full+json" \
+ -H "Content-Type: application/json" \
+ -H "${githubauth}" \
+ -d @"${restfile}" \
+ --silent --location \
+ "${GITHUB_API_URL}/repos/${GITHUB_REPO}/issues/${GITHUB_ISSUE}/comments" \
+ >/dev/null
+
+ retval=$?
+ rm "${restfile}"
return ${retval}
}
-
## @description Print out the finished details to the Github PR
## @audience private
## @stability evolving
@@ -38,14 +418,15 @@ function github_write_comment
## @param runresult
function github_finalreport
{
- local result=$1
- local i
- local commentfile=${PATCH_DIR}/commentfile
- local comment
+ declare result=$1
+ declare i
+ declare commentfile=${PATCH_DIR}/gitcommentfile.$$
+ declare comment
rm "${commentfile}" 2>/dev/null
- if [[ ${JENKINS} != "true" ]] ; then
+ if [[ ${JENKINS} != "true"
+ || -z ${GITHUB_ISSUE} ]] ; then
return 0
fi
@@ -54,15 +435,15 @@ function github_finalreport
add_footer_table "Console output" "${BUILD_URL}console"
if [[ ${result} == 0 ]]; then
- add_header_line ":confetti_ball: **+1 overall**"
+ echo ":confetti_ball: **+1 overall**" >> "${commentfile}"
else
- add_header_line ":broken_heart: **-1 overall**"
+ echo ":broken_heart: **-1 overall**" >> "${commentfile}"
fi
printf "\n\n\n\n" >> "${commentfile}"
i=0
- until [[ $i -eq ${#TP_HEADER[@]} ]]; do
+ until [[ ${i} -eq ${#TP_HEADER[@]} ]]; do
printf "%s\n\n" "${TP_HEADER[${i}]}" >> "${commentfile}"
((i=i+1))
done
@@ -74,7 +455,7 @@ function github_finalreport
} >> "${commentfile}"
i=0
- until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
+ until [[ ${i} -eq ${#TP_VOTE_TABLE[@]} ]]; do
echo "${TP_VOTE_TABLE[${i}]}" >> "${commentfile}"
((i=i+1))
done
@@ -86,7 +467,7 @@ function github_finalreport
echo "|-------:|:------|"
} >> "${commentfile}"
i=0
- until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do
+ until [[ ${i} -eq ${#TP_TEST_TABLE[@]} ]]; do
echo "${TP_TEST_TABLE[${i}]}" >> "${commentfile}"
((i=i+1))
done
@@ -108,5 +489,5 @@ function github_finalreport
printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}"
- write_to_github "${commentfile}"
+ github_write_comment "${commentfile}"
}
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/test-patch.d/jira.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/jira.sh b/dev-support/test-patch.d/jira.sh
index f95ca6f..ca9f2bd 100755
--- a/dev-support/test-patch.d/jira.sh
+++ b/dev-support/test-patch.d/jira.sh
@@ -14,26 +14,37 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-JIRACLI=${JIRA:-jira}
+# this bug system handles JIRA. Personalities
+# can override the following variables:
+
+# base JIRA URL
+JIRA_URL=${JIRA_URL:-"https://issues.apache.org/jira"}
+
+# Issue regex to help identify the project
+JIRA_ISSUE_RE=''
add_bugsystem jira
function jira_usage
{
echo "JIRA Options:"
- echo "--jira-cmd=<cmd> The 'jira' command to use (default 'jira')"
- echo "--jira-password=<pw> The password for the 'jira' command"
- echo "--jira-user=<user> The user for the 'jira' command"
+ echo "--jira-issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default: \'${JIRA_ISSUE_RE}\')"
+ echo "--jira-password=<pw> The password for the 'jira' command"
+ echo "--jira-base-url=<url> The URL of the JIRA server (default:'${JIRA_URL}')"
+ echo "--jira-user=<user> The user for the 'jira' command"
}
function jira_parse_args
{
- local i
+ declare i
for i in "$@"; do
case ${i} in
- --jira-cmd=*)
- JIRACLI=${i#*=}
+ --jira-base-url=*)
+ JIRA_URL=${i#*=}
+ ;;
+ --jira-issue-re=*)
+ JIRA_ISSUE_RE=${i#*=}
;;
--jira-password=*)
JIRA_PASSWD=${i#*=}
@@ -45,31 +56,248 @@ function jira_parse_args
done
}
+## @description provides issue determination based upon the URL and more.
+## @description WARNING: called from the github plugin!
+function jira_determine_issue
+{
+ declare input=$1
+ declare patchnamechunk
+ declare maybeissue
+
+ # shellcheck disable=SC2016
+ patchnamechunk=$(echo "${input}" | ${AWK} -F/ '{print $NF}')
+
+ maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-)
+
+ if [[ ${maybeissue} =~ ${JIRA_ISSUE_RE} ]]; then
+ ISSUE=${maybeissue}
+ JIRA_ISSUE=${maybeissue}
+ add_footer_table "JIRA Issue" "${ISSUE}"
+ return 0
+ fi
+
+ return 1
+}
+
+function jira_http_fetch
+{
+ declare input=$1
+ declare output=$2
+
+ if [[ -n "${JIRA_USER}"
+ && -n "${JIRA_PASSWD}" ]]; then
+ ${CURL} --silent --fail \
+ --user "${JIRA_USER}:${JIRA_PASSWD}" \
+ --output "${output}" \
+ --location \
+ "${JIRA_URL}/${input}"
+ else
+ ${CURL} --silent --fail \
+ --output "${output}" \
+ --location \
+ "${JIRA_URL}/${input}"
+ fi
+}
+
+function jira_locate_patch
+{
+ declare input=$1
+ declare fileloc=$2
+ declare relativeurl
+
+ yetus_debug "jira_locate_patch: trying ${JIRA_URL}/browse/${input}"
+
+ if [[ "${OFFLINE}" == true ]]; then
+ yetus_debug "jira_locate_patch: offline, skipping"
+ return 1
+ fi
+
+ jira_http_fetch "browse/${input}" "${PATCH_DIR}/jira"
+
+ if [[ $? != 0 ]]; then
+ yetus_debug "jira_locate_patch: not a JIRA."
+ return 1
+ fi
+
+ # if github is configured and we see what looks like a URL,
+ # send this to the github plugin to process.
+ if [[ -n "${GITHUB_BASE_URL}"
+ && $(${GREP} -c "${GITHUB_BASE_URL}"'[^ ]*patch' "${PATCH_DIR}/jira") != 0 ]]; then
+ echo "${input} appears to be a Github PR. Switching Modes."
+ github_jira_bridge "${fileloc}"
+ return $?
+ elif [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]]; then
+ if [[ ${JENKINS} == true ]]; then
+ yetus_error "ERROR: ${input} is not \"Patch Available\"."
+ cleanup_and_exit 1
+ else
+ yetus_error "WARNING: ${input} is not \"Patch Available\"."
+ fi
+ fi
+
+ #shellcheck disable=SC2016
+ relativeurl=$(${AWK} 'match($0,"/secure/attachment/[0-9]*/[^\"]*"){print substr($0,RSTART,RLENGTH)}' "${PATCH_DIR}/jira" |
+ ${GREP} -v -e 'htm[l]*$' | sort | tail -1 | ${SED} -e 's,[ ]*$,,g')
+ PATCHURL="${JIRA_URL}${relativeurl}"
+ if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
+ guess_patch_file "${PATCH_DIR}/patch"
+ if [[ $? == 0 ]]; then
+ yetus_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. Proceeding, but issue/branch matching might go awry."
+ add_vote_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
+ fi
+ fi
+ echo "${input} patch is being downloaded at $(date) from"
+ echo "${PATCHURL}"
+ add_footer_table "JIRA Patch URL" "${PATCHURL}"
+ jira_http_fetch "${relativeurl}" "${fileloc}"
+ if [[ $? != 0 ]];then
+ yetus_error "ERROR: ${input}/${PATCHURL} could not be downloaded."
+ cleanup_and_exit 1
+ fi
+ return 0
+}
+
+## @description Try to guess the branch being tested using a variety of heuristics
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success, with PATCH_BRANCH updated appropriately
+function jira_determine_branch
+{
+ declare patchnamechunk
+ declare total
+ declare count
+ declare hinttype
+
+ for hinttype in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
+ if [[ -z "${hinttype}" ]]; then
+ continue
+ fi
+
+ # If one of these matches the JIRA issue regex
+ # then we don't want it to trigger the branch
+ # detection since that's almost certainly not
+ # intended. In other words, if ISSUE-99 is the
+ # name of a branch, you want to test ISSUE-99
+ # against master, not ISSUE-99's branch
+ if [[ ${hinttype} =~ ${JIRA_ISSUE_RE} ]]; then
+ continue
+ fi
+
+ yetus_debug "Determine branch: starting with ${hinttype}"
+ patchnamechunk=$(echo "${hinttype}" \
+ | ${SED} -e 's,.*/\(.*\)$,\1,' \
+ -e 's,\.txt,.,' \
+ -e 's,.patch,.,g' \
+ -e 's,.diff,.,g' \
+ -e 's,\.\.,.,g' \
+ -e 's,\.$,,g' )
+
+ # ISSUE-branch-##
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
+ yetus_debug "Determine branch: ISSUE-branch-## = ${PATCH_BRANCH}"
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+ fi
+
+ # ISSUE-##[.##].branch
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d. )
+ count="${PATCH_BRANCH//[^.]}"
+ total=${#count}
+ ((total = total + 3 ))
+ until [[ ${total} -lt 2 ]]; do
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3-${total} -d.)
+ yetus_debug "Determine branch: ISSUE[.##].branch = ${PATCH_BRANCH}"
+ ((total=total-1))
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+ fi
+ done
+
+ # ISSUE.branch.##
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2- -d. )
+ count="${PATCH_BRANCH//[^.]}"
+ total=${#count}
+ ((total = total + 3 ))
+ until [[ ${total} -lt 2 ]]; do
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2-${total} -d.)
+ yetus_debug "Determine branch: ISSUE.branch[.##] = ${PATCH_BRANCH}"
+ ((total=total-1))
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+ fi
+ done
+
+ # ISSUE-branch.##
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1- -d. )
+ count="${PATCH_BRANCH//[^.]}"
+ total=${#count}
+ ((total = total + 1 ))
+ until [[ ${total} -eq 1 ]]; do
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1-${total} -d. )
+ yetus_debug "Determine branch: ISSUE-branch[.##] = ${PATCH_BRANCH}"
+ ((total=total-1))
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+ fi
+ done
+ done
+
+ return 1
+}
+
## @description Write the contents of a file to JIRA
## @params filename
## @stability stable
## @audience public
-## @returns ${JIRACLI} exit code
+## @returns exit code from posting to jira
function jira_write_comment
{
- local -r commentfile=${1}
- shift
-
- local retval=0
+ declare -r commentfile=${1}
+ declare retval=0
+ if [[ "${OFFLINE}" == true ]]; then
+ echo "JIRA Plugin: Running in offline, comment skipped."
+ return 0
+ fi
if [[ -n ${JIRA_PASSWD}
&& -n ${JIRA_USER} ]]; then
- # shellcheck disable=SC2086
- ${JIRACLI} --comment "$(cat ${commentfile})" \
- -s https://issues.apache.org/jira \
- -a addcomment -u ${JIRA_USER} \
- -p "${JIRA_PASSWD}" \
- --issue "${ISSUE}"
+
+ # RESTify the comment
+ {
+ echo "{\"body\":\""
+ ${SED} -e 's,\\,\\\\,g' \
+ -e 's,\",\\\",g' \
+ -e 's,$,\\r\\n,g' "${commentfile}" \
+ | tr -d '\n'
+ echo "\"}"
+ } > "${PATCH_DIR}/jiracomment.$$"
+
+ ${CURL} -X POST \
+ -H "Accept: application/json" \
+ -H "Content-Type: application/json" \
+ -u "${JIRA_USER}:${JIRA_PASSWD}" \
+ -d @"${PATCH_DIR}/jiracomment.$$" \
+ --silent --location \
+ "${JIRA_URL}/rest/api/2/issue/${JIRA_ISSUE}/comment" \
+ >/dev/null
retval=$?
- ${JIRACLI} -s https://issues.apache.org/jira \
- -a logout -u "${JIRA_USER}" \
- -p "${JIRA_PASSWD}"
+ rm "${PATCH_DIR}/jiracomment.$$"
+ else
+ echo "JIRA Plugin: no credentials provided to write a comment."
fi
return ${retval}
}
@@ -81,20 +309,25 @@ function jira_write_comment
## @param runresult
function jira_finalreport
{
- local result=$1
- local i
- local commentfile=${PATCH_DIR}/commentfile
- local comment
- local vote
- local ourstring
- local ela
- local subs
- local color
- local comment
+ declare result=$1
+ declare i
+ declare commentfile=${PATCH_DIR}/jiracommentfile
+ declare comment
+ declare vote
+ declare ourstring
+ declare ela
+ declare subs
+ declare color
+ declare comment
rm "${commentfile}" 2>/dev/null
- if [[ ${JENKINS} != "true" ]] ; then
+ if [[ ${JENKINS} == "false"
+ || ${OFFLINE} == true ]] ; then
+ return 0
+ fi
+
+ if [[ -z "${JIRA_ISSUE}" ]]; then
return 0
fi
@@ -103,12 +336,12 @@ function jira_finalreport
add_footer_table "Console output" "${BUILD_URL}console"
if [[ ${result} == 0 ]]; then
- add_header_line "| (/) *{color:green}+1 overall{color}* |"
+ echo "| (/) *{color:green}+1 overall{color}* |" >> "${commentfile}"
else
- add_header_line "| (x) *{color:red}-1 overall{color}* |"
+ echo "| (x) *{color:red}-1 overall{color}* |" >> "${commentfile}"
fi
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
+ echo "\\\\" >> "${commentfile}"
i=0
until [[ $i -eq ${#TP_HEADER[@]} ]]; do
@@ -116,7 +349,7 @@ function jira_finalreport
((i=i+1))
done
- { echo "\\\\" ; echo "\\\\"; } >> "${commentfile}"
+ echo "\\\\" >> "${commentfile}"
echo "|| Vote || Subsystem || Runtime || Comment ||" >> "${commentfile}"
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
index 4d17768..0c198db 100755
--- a/dev-support/test-patch.d/shellcheck.sh
+++ b/dev-support/test-patch.d/shellcheck.sh
@@ -137,7 +137,10 @@ function shellcheck_postapply
fi
add_footer_table shellcheck "${msg}"
- calcdiffs "${PATCH_DIR}/branch-shellcheck-result.txt" "${PATCH_DIR}/patch-shellcheck-result.txt" > "${PATCH_DIR}/diff-patch-shellcheck.txt"
+ calcdiffs \
+ "${PATCH_DIR}/branch-shellcheck-result.txt" \
+ "${PATCH_DIR}/patch-shellcheck-result.txt" \
+ > "${PATCH_DIR}/diff-patch-shellcheck.txt"
# shellcheck disable=SC2016
diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-shellcheck.txt" | ${AWK} '{print $1}')
@@ -151,6 +154,7 @@ function shellcheck_postapply
add_vote_table -1 shellcheck "The applied patch generated "\
"${diffPostpatch} new shellcheck issues (total was ${numPrepatch}, now ${numPostpatch})."
add_footer_table shellcheck "@@BASE@@/diff-patch-shellcheck.txt"
+ bugsystem_linecomments "shellcheck" "${PATCH_DIR}/diff-patch-shellcheck.txt"
return 1
fi
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/test-patch.d/whitespace.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/whitespace.sh b/dev-support/test-patch.d/whitespace.sh
index 6fc033b..bab32dd 100755
--- a/dev-support/test-patch.d/whitespace.sh
+++ b/dev-support/test-patch.d/whitespace.sh
@@ -16,6 +16,26 @@
add_plugin whitespace
+
+function whitespace_linecomment_reporter
+{
+ local file=$1
+ shift
+ local comment=$*
+ local tmpfile="${PATCH_DIR}/wlr.$$.${RANDOM}"
+
+ while read -r line; do
+ {
+ #shellcheck disable=SC2086
+ printf "%s" "$(echo ${line} | cut -f1-2 -d:)"
+ echo "${comment}"
+ } >> "${tmpfile}"
+ done < "${file}"
+
+ bugsystem_linecomments "whitespace:" "${tmpfile}"
+ rm "${tmpfile}"
+}
+
function whitespace_postapply
{
local count
@@ -40,6 +60,8 @@ function whitespace_postapply
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 whitespace "The patch has ${count}"\
" line(s) that end in whitespace. Use git apply --whitespace=fix."
+
+ whitespace_linecomment_reporter "${PATCH_DIR}/whitespace-eol.txt" "end of line"
add_footer_table whitespace "@@BASE@@/whitespace-eol.txt"
((result=result+1))
fi
@@ -51,6 +73,7 @@ function whitespace_postapply
add_vote_table -1 whitespace "The patch has ${count}"\
" line(s) with tabs."
add_footer_table whitespace "@@BASE@@/whitespace-tabs.txt"
+ whitespace_linecomment_reporter "${PATCH_DIR}/whitespace-tabs.txt" "tabs in line"
((result=result+1))
fi
http://git-wip-us.apache.org/repos/asf/yetus/blob/463891f3/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index a368f83..67dc3b4 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -98,23 +98,18 @@ function setup_defaults
REEXECED=false
RESETREPO=false
ISSUE=""
- ISSUE_RE='^(YETUS)-[0-9]+$'
TIMER=$(date +"%s")
- PATCHURL=""
OSTYPE=$(uname -s)
BUILDTOOL=maven
- BUGSYSTEM=jira
TESTFORMATS=""
JDK_TEST_LIST="javac javadoc unit"
- GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt"
- GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
# Solaris needs POSIX, not SVID
case ${OSTYPE} in
SunOS)
AWK=${AWK:-/usr/xpg4/bin/awk}
SED=${SED:-/usr/xpg4/bin/sed}
- WGET=${WGET:-wget}
+ CURL=${CURL:-curl}
GIT=${GIT:-git}
GREP=${GREP:-/usr/xpg4/bin/grep}
PATCH=${PATCH:-/usr/gnu/bin/patch}
@@ -124,7 +119,7 @@ function setup_defaults
*)
AWK=${AWK:-awk}
SED=${SED:-sed}
- WGET=${WGET:-wget}
+ CURL=${CURL:-curl}
GIT=${GIT:-git}
GREP=${GREP:-grep}
PATCH=${PATCH:-patch}
@@ -242,6 +237,7 @@ function offset_clock
## @param string
function add_header_line
{
+ # shellcheck disable=SC2034
TP_HEADER[${TP_HEADER_COUNTER}]="$*"
((TP_HEADER_COUNTER=TP_HEADER_COUNTER+1 ))
}
@@ -279,8 +275,10 @@ function add_vote_table
fi
if [[ -z ${value} ]]; then
+ # shellcheck disable=SC2034
TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | ${subsystem} | | ${*:-} |"
else
+ # shellcheck disable=SC2034
TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${calctime} | $* |"
fi
((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1))
@@ -423,6 +421,7 @@ function finish_vote_table
echo "Total Elapsed time: ${calctime}"
echo ""
+ # shellcheck disable=SC2034
TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${calctime} | |"
((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1 ))
}
@@ -440,6 +439,7 @@ function add_footer_table
local subsystem=$1
shift 1
+ # shellcheck disable=SC2034
TP_FOOTER_TABLE[${TP_FOOTER_COUNTER}]="| ${subsystem} | $* |"
((TP_FOOTER_COUNTER=TP_FOOTER_COUNTER+1 ))
}
@@ -455,6 +455,7 @@ function add_test_table
local failure=$1
shift 1
+ # shellcheck disable=SC2034
TP_TEST_TABLE[${TP_TEST_COUNTER}]="| ${failure} | $* |"
((TP_TEST_COUNTER=TP_TEST_COUNTER+1 ))
}
@@ -538,24 +539,21 @@ function find_java_home
return 0
}
-## @description Write the contents of a file to jenkins
+## @description Write the contents of a file to all of the bug systems
+## @description (so content should avoid special formatting)
## @params filename
## @stability stable
## @audience public
-## @returns ${JIRACLI} exit code
function write_comment
{
local -r commentfile=${1}
- shift
+ declare bug
- local retval=0
-
- if [[ ${OFFLINE} == false
- && ${JENKINS} == true ]]; then
- ${BUGSYSTEM}_write_comment "${commentfile}"
- retval=$?
- fi
- return ${retval}
+ for bug in ${BUGCOMMENTS}; do
+ if declare -f ${bug}_write_comment >/dev/null; then
+ "${bug}_write_comment" "${commentfile}"
+ fi
+ done
}
## @description Verify that the patch directory is still in working order
@@ -611,7 +609,7 @@ function compute_gitdiff
pushd "${BASEDIR}" >/dev/null
${GIT} add --all --intent-to-add
- while read line; do
+ while read -r line; do
if [[ ${line} =~ ^\+\+\+ ]]; then
file="./"$(echo "${line}" | cut -f2- -d/)
continue
@@ -651,16 +649,70 @@ function compute_gitdiff
fi
done < <("${GIT}" diff --unified=0 --no-color)
- if [[ ! -f ${GITDIFFLINES} ]]; then
+ if [[ ! -f "${GITDIFFLINES}" ]]; then
touch "${GITDIFFLINES}"
fi
- if [[ ! -f ${GITDIFFCONTENT} ]]; then
+
+ if [[ ! -f "${GITDIFFCONTENT}" ]]; then
touch "${GITDIFFCONTENT}"
fi
+ if [[ -s "${GITDIFFLINES}" ]]; then
+ compute_unidiff
+ else
+ touch "${GITUNIDIFFLINES}"
+ fi
+
popd >/dev/null
}
+## @description generate an index of unified diff lines vs. modified/added lines
+## @description ${GITDIFFLINES} must exist.
+## @audience private
+## @stability stable
+## @replaceable no
+function compute_unidiff
+{
+ declare fn
+ declare filen
+ declare tmpfile="${PATCH_DIR}/tmp.$$.${RANDOM}"
+
+ # now that we know what lines are where, we can deal
+ # with github's pain-in-the-butt API. It requires
+ # that the client provides the line number of the
+ # unified diff on a per file basis.
+
+ # First, build a per-file unified diff, pulling
+ # out the 'extra' lines, grabbing the adds with
+ # the line number in the diff file along the way,
+ # finally rewriting the line so that it is in
+ # './filename:diff line:content' format
+
+ for fn in ${CHANGED_FILES}; do
+ filen=${fn##./}
+
+ ${GIT} diff ${filen} \
+ | tail -n +6 \
+ | ${GREP} -n '^+' \
+ | ${GREP} -vE '^[0-9]*:\+\+\+' \
+ | ${SED} -e 's,^\([0-9]*:\)\+,\1,g' \
+ -e s,^,./${filen}:,g \
+ >> "${tmpfile}"
+ done
+
+ # at this point, tmpfile should be in the same format
+ # as gitdiffcontent, just with different line numbers.
+ # let's do a merge (using gitdifflines because it's easier)
+
+ # ./filename:real number:diff number
+ # shellcheck disable=SC2016
+ paste -d: "${GITDIFFLINES}" "${tmpfile}" \
+ | ${AWK} -F: '{print $1":"$2":"$5":"$6}' \
+ >> "${GITUNIDIFFLINES}"
+
+ rm "${tmpfile}"
+}
+
## @description Print the command to be executing to the screen. Then
## @description run the command, sending stdout and stderr to the given filename
## @description This will also ensure that any directories in ${BASEDIR} have
@@ -729,17 +781,16 @@ function testpatch_usage
echo "--basedir=<dir> The directory to apply the patch to (default current directory)"
echo "--branch=<ref> Forcibly set the branch"
echo "--branch-default=<ref> If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'master')"
- #not quite working yet
- #echo "--bugsystem=<type> The bug system in use ('jira', the default, or 'github')"
echo "--build-native=<bool> If true, then build native components (default 'true')"
echo "--build-tool=<tool> Pick which build tool to focus around (maven, ant)"
+ echo "--bugcomments=<bug> Only write comments to the screen and this comma delimited list"
echo "--contrib-guide=<url> URL to point new users towards project conventions. (default: ${HOW_TO_CONTRIBUTE} )"
echo "--debug If set, then output some extra stuff to stderr"
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
echo "--docker Spawn a docker container"
echo "--dockerfile=<file> Dockerfile fragment to use as the base"
- echo "--issue-re=<expr> Bash regular expression to use when trying to find a jira ref in the patch name (default: \'${ISSUE_RE}\')"
echo "--java-home=<path> Set JAVA_HOME (In Docker mode, this should be local to the image)"
+ echo "--linecomments=<bug> Only write line comments to this comma delimited list (defaults to bugcomments)"
echo "--multijdkdirs=<paths> Comma delimited lists of JDK paths to use for multi-JDK tests"
echo "--multijdktests=<list> Comma delimited tests to use when multijdkdirs is used. (default: javac,javadoc,unit)"
echo "--modulelist=<list> Specify additional modules to test (comma delimited)"
@@ -760,6 +811,7 @@ function testpatch_usage
echo "Shell binary overrides:"
echo "--ant-cmd=<cmd> The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')"
echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
+ echo "--curl-cmd=<cmd> The 'wget' command to use (default 'curl')"
echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')"
echo "--file-cmd=<cmd> The 'file' command to use (default 'file')"
echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
@@ -774,7 +826,6 @@ function testpatch_usage
echo "--build-url Set the build location web page"
echo "--eclipse-home=<path> Eclipse home directory (default ECLIPSE_HOME environment variable)"
echo "--mv-patch-dir Move the patch-dir into the basedir during cleanup."
- echo "--wget-cmd=<cmd> The 'wget' command to use (default 'wget')"
importplugins
@@ -815,8 +866,9 @@ function parse_args
--branch-default=*)
PATCH_BRANCH_DEFAULT=${i#*=}
;;
- --bugsystem=*)
- BUGSYSTEM=${i#*=}
+ --bugcomments=*)
+ BUGCOMMENTS=${i#*=}
+ BUGCOMMENTS=${BUGCOMMENTS//,/ }
;;
--build-native=*)
BUILD_NATIVE=${i#*=}
@@ -830,6 +882,9 @@ function parse_args
--contrib-guide=*)
HOW_TO_CONTRIBUTE=${i#*=}
;;
+ --curl-cmd=*)
+ CURL=${i#*=}
+ ;;
--debug)
TP_SHELL_SCRIPT_DEBUG=true
;;
@@ -864,9 +919,6 @@ function parse_args
testpatch_usage
exit 0
;;
- --issue-re=*)
- ISSUE_RE=${i#*=}
- ;;
--java-home=*)
JAVA_HOME=${i#*=}
;;
@@ -874,6 +926,10 @@ function parse_args
JENKINS=true
TEST_PARALLEL=${TEST_PARALLEL:-true}
;;
+ --linecomments=*)
+ BUGLINECOMMENTS=${i#*=}
+ BUGLINECOMMENTS=${BUGLINECOMMENTS//,/ }
+ ;;
--modulelist=*)
USER_MODULE_LIST=${i#*=}
USER_MODULE_LIST=${USER_MODULE_LIST//,/ }
@@ -954,9 +1010,6 @@ function parse_args
--tpreexectimer=*)
REEXECLAUNCHTIMER=${i#*=}
;;
- --wget-cmd=*)
- WGET=${i#*=}
- ;;
--*)
## PATCH_OR_ISSUE can't be a --. So this is probably
## a plugin thing.
@@ -1039,6 +1092,8 @@ function parse_args
GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt"
GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
+ GITUNIDIFFLINES="${PATCH_DIR}/gitdiffunilines.txt"
+
}
## @description Locate the build file for a given directory
@@ -1148,8 +1203,7 @@ function find_changed_modules
;;
*)
yetus_error "ERROR: Unsupported build tool."
- output_to_console 1
- output_to_bugsystem 1
+ bugsystem_finalreport 1
cleanup_and_exit 1
;;
esac
@@ -1167,8 +1221,7 @@ function find_changed_modules
builddir=$(find_buildfile_dir ${buildfile} "${i}")
if [[ -z ${builddir} ]]; then
yetus_error "ERROR: ${buildfile} is not found. Make sure the target is a ${BUILDTOOL}-based project."
- output_to_console 1
- output_to_bugsystem 1
+ bugsystem_finalreport 1
cleanup_and_exit 1
fi
builddirs="${builddirs} ${builddir}"
@@ -1188,6 +1241,8 @@ function find_changed_modules
buildmods="${buildmods} ${module}"
fi
done
+ else
+ buildmods=${CHANGED_UNFILTERED_MODULES}
fi
#shellcheck disable=SC2086,SC2034
@@ -1245,6 +1300,7 @@ function git_checkout
{
local currentbranch
local exemptdir
+ local status
big_console_header "Confirming git environment"
@@ -1342,8 +1398,6 @@ function git_checkout
determine_issue
GIT_REVISION=$(${GIT} rev-parse --verify --short HEAD)
- # shellcheck disable=SC2034
- VERSION=${GIT_REVISION}_${ISSUE}_PATCH-${patchNum}
if [[ "${ISSUE}" == 'Unknown' ]]; then
echo "Testing patch on ${PATCH_BRANCH}."
@@ -1402,9 +1456,8 @@ function verify_valid_branch
## @return 1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT
function determine_branch
{
- local patchnamechunk
- local total
- local count
+ declare bugs
+ declare retval=1
# something has already set this, so move on
if [[ -n ${PATCH_BRANCH} ]]; then
@@ -1427,83 +1480,19 @@ function determine_branch
return
fi
- for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
- if [[ -z "${j}" ]]; then
- continue
- fi
- yetus_debug "Determine branch: starting with ${j}"
- patchnamechunk=$(echo "${j}" \
- | ${SED} -e 's,.*/\(.*\)$,\1,' \
- -e 's,\.txt,.,' \
- -e 's,.patch,.,g' \
- -e 's,.diff,.,g' \
- -e 's,\.\.,.,g' \
- -e 's,\.$,,g' )
-
- # ISSUE-branch-##
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
- yetus_debug "Determine branch: ISSUE-branch-## = ${PATCH_BRANCH}"
- if [[ -n "${PATCH_BRANCH}" ]]; then
- verify_valid_branch "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
+ for bugs in ${BUGSYSTEMS}; do
+ if declare -f ${bugs}_determine_branch >/dev/null;then
+ "${bugs}_determine_branch"
+ retval=$?
+ if [[ ${retval} == 0 ]]; then
+ break
fi
fi
-
- # ISSUE-##[.##].branch
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d. )
- count="${PATCH_BRANCH//[^.]}"
- total=${#count}
- ((total = total + 3 ))
- until [[ ${total} -eq 2 ]]; do
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3-${total} -d.)
- yetus_debug "Determine branch: ISSUE[.##].branch = ${PATCH_BRANCH}"
- ((total=total-1))
- if [[ -n "${PATCH_BRANCH}" ]]; then
- verify_valid_branch "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
- fi
- done
-
- # ISSUE.branch.##
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2- -d. )
- count="${PATCH_BRANCH//[^.]}"
- total=${#count}
- ((total = total + 3 ))
- until [[ ${total} -eq 2 ]]; do
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2-${total} -d.)
- yetus_debug "Determine branch: ISSUE.branch[.##] = ${PATCH_BRANCH}"
- ((total=total-1))
- if [[ -n "${PATCH_BRANCH}" ]]; then
- verify_valid_branch "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
- fi
- done
-
- # ISSUE-branch.##
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1- -d. )
- count="${PATCH_BRANCH//[^.]}"
- total=${#count}
- ((total = total + 1 ))
- until [[ ${total} -eq 1 ]]; do
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1-${total} -d. )
- yetus_debug "Determine branch: ISSUE-branch[.##] = ${PATCH_BRANCH}"
- ((total=total-1))
- if [[ -n "${PATCH_BRANCH}" ]]; then
- verify_valid_branch "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
- fi
- done
-
done
- PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}"
+ if [[ ${retval} != 0 ]]; then
+ PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}"
+ fi
popd >/dev/null
}
@@ -1515,28 +1504,19 @@ function determine_branch
## @return 1 on failure, with ISSUE updated to "Unknown"
function determine_issue
{
- local patchnamechunk
- local maybeissue
+ local bugsys
yetus_debug "Determine issue"
- # we can shortcut jenkins
- if [[ ${JENKINS} == true ]]; then
- ISSUE=${PATCH_OR_ISSUE}
- return 0
- fi
-
- # shellcheck disable=SC2016
- patchnamechunk=$(echo "${PATCH_OR_ISSUE}" | ${AWK} -F/ '{print $NF}')
-
- maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-)
-
- if [[ ${maybeissue} =~ ${ISSUE_RE} ]]; then
- ISSUE=${maybeissue}
- return 0
- fi
-
- ISSUE="Unknown"
+ for bugsys in ${BUGSYSTEMS}; do
+ if declare -f ${bugsys}_determine_issue >/dev/null; then
+ "${bugsys}_determine_issue" "${PATCH_OR_ISSUE}"
+ if [[ $? == 0 ]]; then
+ yetus_debug "${bugsys} says ${ISSUE}"
+ return 0
+ fi
+ fi
+ done
return 1
}
@@ -1585,6 +1565,7 @@ function verify_needed_test
function determine_needed_tests
{
local i
+ local plugin
for i in ${CHANGED_FILES}; do
yetus_debug "Determining needed tests for ${i}"
@@ -1609,100 +1590,51 @@ function determine_needed_tests
## @return 1 on failure, may exit
function locate_patch
{
- local notSureIfPatch=false
+ local bugsys
+ local patchfile=""
+ local gotit=false
+
yetus_debug "locate patch"
+ # it's a locally provided file
if [[ -f ${PATCH_OR_ISSUE} ]]; then
- PATCH_FILE="${PATCH_OR_ISSUE}"
+ patchfile="${PATCH_OR_ISSUE}"
else
- if [[ ${PATCH_OR_ISSUE} =~ ^http ]]; then
- echo "Patch is being downloaded at $(date) from"
- PATCHURL="${PATCH_OR_ISSUE}"
- else
- ${WGET} -q -O "${PATCH_DIR}/jira" "http://issues.apache.org/jira/browse/${PATCH_OR_ISSUE}"
-
- case $? in
- 0)
- ;;
- 2)
- yetus_error "ERROR: .wgetrc/.netrc parsing error."
- cleanup_and_exit 1
- ;;
- 3)
- yetus_error "ERROR: File IO error."
- cleanup_and_exit 1
- ;;
- 4)
- yetus_error "ERROR: URL ${PATCH_OR_ISSUE} is unreachable."
- cleanup_and_exit 1
- ;;
- *)
- # we want to try and do as much as we can in docker mode,
- # but if the patch was passed as a file, then we may not
- # be able to continue.
- if [[ ${REEXECED} == true
- && -f "${PATCH_DIR}/patch" ]]; then
- PATCH_FILE="${PATCH_DIR}/patch"
- else
- yetus_error "ERROR: Unable to fetch ${PATCH_OR_ISSUE}."
- cleanup_and_exit 1
- fi
- ;;
- esac
-
- if [[ -z "${PATCH_FILE}" ]]; then
- if [[ $(${GREP} -c 'Patch Available' "${PATCH_DIR}/jira") == 0 ]] ; then
- if [[ ${JENKINS} == true ]]; then
- yetus_error "ERROR: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
- cleanup_and_exit 1
- else
- yetus_error "WARNING: ${PATCH_OR_ISSUE} is not \"Patch Available\"."
+ # run through the bug systems. maybe they know?
+ for bugsys in ${BUGSYSTEMS}; do
+ if declare -f ${bugsys}_locate_patch >/dev/null 2>&1; then
+ "${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
+ if [[ $? == 0 ]]; then
+ guess_patch_file "${PATCH_DIR}/patch"
+ if [[ $? == 0 ]]; then
+ gotit=true
+ break;
fi
fi
-
- #shellcheck disable=SC2016
- relativePatchURL=$(${AWK} 'match($0,"\"/jira/secure/attachment/[0-9]*/[^\"]*"){print substr($0,RSTART+1,RLENGTH-1)}' "${PATCH_DIR}/jira" |
- ${GREP} -v -e 'htm[l]*$' | sort | tail -1)
- PATCHURL="http://issues.apache.org${relativePatchURL}"
- if [[ ! ${PATCHURL} =~ \.patch$ ]]; then
- notSureIfPatch=true
- fi
- #shellcheck disable=SC2016
- patchNum=$(echo "${PATCHURL}" | ${AWK} 'match($0,"[0-9]*/"){print substr($0,RSTART,RLENGTH-1)}')
- echo "${ISSUE} patch is being downloaded at $(date) from"
fi
- fi
- if [[ -z "${PATCH_FILE}" ]]; then
- echo "${PATCHURL}"
- add_footer_table "Patch URL" "${PATCHURL}"
- ${WGET} -q -O "${PATCH_DIR}/patch" "${PATCHURL}"
- if [[ $? != 0 ]];then
- yetus_error "ERROR: ${PATCH_OR_ISSUE} could not be downloaded."
- cleanup_and_exit 1
- fi
- PATCH_FILE="${PATCH_DIR}/patch"
+ done
+
+ # ok, none of the bug systems know. let's see how smart we are
+ if [[ ${gotit} == false ]]; then
+ generic_locate_patch "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
fi
fi
- if [[ ! -f "${PATCH_DIR}/patch" ]]; then
- cp "${PATCH_FILE}" "${PATCH_DIR}/patch"
+ if [[ ! -f "${PATCH_DIR}/patch"
+ && -f "${patchfile}" ]]; then
+ cp "${patchfile}" "${PATCH_DIR}/patch"
if [[ $? == 0 ]] ; then
- echo "Patch file ${PATCH_FILE} copied to ${PATCH_DIR}"
+ echo "Patch file ${patchfile} copied to ${PATCH_DIR}"
else
- yetus_error "ERROR: Could not copy ${PATCH_FILE} to ${PATCH_DIR}"
+ yetus_error "ERROR: Could not copy ${patchfile} to ${PATCH_DIR}"
cleanup_and_exit 1
fi
fi
- if [[ ${notSureIfPatch} == "true" ]]; then
- guess_patch_file "${PATCH_DIR}/patch"
- if [[ $? != 0 ]]; then
- yetus_error "ERROR: ${PATCHURL} is not a patch file."
- cleanup_and_exit 1
- else
- yetus_debug "The patch ${PATCHURL} was not named properly, but it looks like a patch file. proceeding, but issue/branch matching might go awry."
- add_vote_table 0 patch "The patch file was not named according to ${PROJECT_NAME}'s naming conventions. Please see ${HOW_TO_CONTRIBUTE} for instructions."
- fi
+ guess_patch_file "${PATCH_DIR}/patch"
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}."
+ cleanup_and_exit 1
fi
}
@@ -1717,6 +1649,10 @@ function guess_patch_file
local patch=$1
local fileOutput
+ if [[ ! -f ${patch} ]]; then
+ return 1
+ fi
+
yetus_debug "Trying to guess is ${patch} is a patch file."
fileOutput=$("${FILE}" "${patch}")
if [[ $fileOutput =~ \ diff\ ]]; then
@@ -1769,8 +1705,7 @@ function apply_patch_file
echo "PATCH APPLICATION FAILED"
((RESULT = RESULT + 1))
add_vote_table -1 patch "The patch command could not apply the patch."
- output_to_console 1
- output_to_bugsystem 1
+ bugsystem_finalreport 1
cleanup_and_exit 1
fi
return 0
@@ -3064,134 +2999,55 @@ function check_unittests
return 0
}
-## @description Print out the finished details on the console
-## @audience private
+## @description Write comments onto bug systems that have code review support.
+## @description File should be in the form of "file:line:comment"
+## @audience public
## @stability evolving
## @replaceable no
-## @param runresult
-## @return 0 on success
-## @return 1 on failure
-function output_to_console
-{
- local result=$1
- shift
- local i=0
- local ourstring
- local vote
- local subs
- local ela
- local comment
- local commentfile1="${PATCH_DIR}/comment.1"
- local commentfile2="${PATCH_DIR}/comment.2"
- local normaltop
- local line
- local seccoladj=0
- local spcfx=${PATCH_DIR}/spcl.txt
-
- if [[ ${result} == 0 ]]; then
- if [[ ${JENKINS} == false ]]; then
- {
- printf "IF9fX19fX19fX18gCjwgU3VjY2VzcyEgPgogLS0tLS0tLS0tLSAKIFwgICAg";
- printf "IC9cICBfX18gIC9cCiAgXCAgIC8vIFwvICAgXC8gXFwKICAgICAoKCAgICBP";
- printf "IE8gICAgKSkKICAgICAgXFwgLyAgICAgXCAvLwogICAgICAgXC8gIHwgfCAg";
- printf "XC8gCiAgICAgICAgfCAgfCB8ICB8ICAKICAgICAgICB8ICB8IHwgIHwgIAog";
- printf "ICAgICAgIHwgICBvICAgfCAgCiAgICAgICAgfCB8ICAgfCB8ICAKICAgICAg";
- printf "ICB8bXwgICB8bXwgIAo"
- } > "${spcfx}"
- fi
- printf "\n\n+1 overall\n\n"
- else
- if [[ ${JENKINS} == false ]]; then
- {
- printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
- printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
- printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
- printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
- printf "ICAgICAgICAgICAgICAgICAK"
- } > "${spcfx}"
- fi
- printf "\n\n-1 overall\n\n"
- fi
-
- if [[ -f ${spcfx} ]]; then
- if which base64 >/dev/null 2>&1; then
- base64 --decode "${spcfx}" 2>/dev/null
- elif which openssl >/dev/null 2>&1; then
- openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
- fi
- echo
- echo
- rm "${spcfx}"
- fi
-
- seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}")
- if [[ ${seccoladj} -lt 10 ]]; then
- seccoladj=10
+## @param filename
+function bugsystem_linecomments
+{
+ declare title=$1
+ declare fn=$2
+ declare line
+ declare bugs
+ declare realline
+ declare text
+ declare idxline
+ declare uniline
+
+ if [[ ! -f "${GITUNIDIFFLINES}" ]]; then
+ return
fi
- seccoladj=$((seccoladj + 2 ))
- i=0
- until [[ $i -eq ${#TP_HEADER[@]} ]]; do
- printf "%s\n" "${TP_HEADER[${i}]}"
- ((i=i+1))
- done
-
- printf "| %s | %*s | %s | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
- echo "============================================================================"
- i=0
- until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
- ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- ela=$(echo "${ourstring}" | cut -f4 -d\|)
- comment=$(echo "${ourstring}" | cut -f5 -d\|)
-
- echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
- normaltop=$(head -1 "${commentfile1}")
- ${SED} -e '1d' "${commentfile1}" > "${commentfile2}"
-
- printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
- "${subs}" "${ela}" "${normaltop}"
- while read line; do
- printf "| | %*s | | %-s\n" ${seccoladj} " " "${line}"
- done < "${commentfile2}"
-
- ((i=i+1))
- rm "${commentfile2}" "${commentfile1}" 2>/dev/null
- done
+ while read -r line;do
+ file=$(echo "${line}" | cut -f1 -d:)
+ realline=$(echo "${line}" | cut -f2 -d:)
+ text=$(echo "${line}" | cut -f3- -d:)
+ idxline="${file}:${realline}:"
+ uniline=$(${GREP} "${idxline}" "${GITUNIDIFFLINES}" | cut -f3 -d: )
- if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
- seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}")
- printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
- i=0
- until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do
- ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ')
- vote=$(echo "${ourstring}" | cut -f2 -d\|)
- subs=$(echo "${ourstring}" | cut -f3 -d\|)
- printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
- ((i=i+1))
+ for bugs in ${BUGLINECOMMENTS}; do
+ if declare -f ${bugs}_linecomments >/dev/null;then
+ "${bugs}_linecomments" "${title}" "${file}" "${realline}" "${uniline}" "${text}"
+ fi
done
- fi
-
- printf "\n\n|| Subsystem || Report/Notes ||\n"
- echo "============================================================================"
- i=0
-
- until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
- comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
- ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
- printf "%s\n" "${comment}"
- ((i=i+1))
- done
+ done < "${fn}"
}
## @description Write the final output to the selected bug system
## @audience private
## @stability evolving
## @replaceable no
-function output_to_bugsystem
+function bugsystem_finalreport
{
- "${BUGSYSTEM}_finalreport" "${@}"
+ declare bugs
+
+ for bugs in ${BUGCOMMENTS}; do
+ if declare -f ${bugs}_finalreport >/dev/null;then
+ "${bugs}_finalreport" "${@}"
+ fi
+ done
}
## @description Clean the filesystem as appropriate and then exit
@@ -3238,8 +3094,7 @@ function postcheckout
(( RESULT = RESULT + $? ))
if [[ ${RESULT} != 0 ]] ; then
- output_to_console 1
- output_to_bugsystem 1
+ bugsystem_finalreport 1
cleanup_and_exit 1
fi
done
@@ -3255,8 +3110,7 @@ function postcheckout
(( RESULT = RESULT + $? ))
if [[ ${RESULT} != 0 ]] ; then
- output_to_console 1
- output_to_bugsystem 1
+ bugsystem_finalreport 1
cleanup_and_exit 1
fi
fi
@@ -3312,8 +3166,7 @@ function postapply
check_patch_javac
retval=$?
if [[ ${retval} -gt 1 ]] ; then
- output_to_console 1
- output_to_bugsystem 1
+ bugsystem_finalreport 1
cleanup_and_exit 1
fi
@@ -3457,6 +3310,13 @@ function parse_args_plugins
(( RESULT = RESULT + $? ))
fi
done
+
+ BUGCOMMENTS=${BUGCOMMENTS:-${BUGSYSTEMS}}
+ if [[ ! ${BUGCOMMENTS} =~ console ]]; then
+ BUGCOMMENTS="${BUGCOMMENTS} console"
+ fi
+
+ BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}}
}
## @description Register test-patch.d plugins
@@ -3588,6 +3448,5 @@ finish_vote_table
finish_footer_table
-output_to_console ${RESULT}
-output_to_bugsystem ${RESULT}
+bugsystem_finalreport ${RESULT}
cleanup_and_exit ${RESULT}
[17/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/b7a0e94a
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/b7a0e94a
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/b7a0e94a
Branch: refs/heads/master
Commit: b7a0e94aff98262770c90b4fa0008492e5d02777
Parents: 11ad56e c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Aug 5 15:40:22 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Aug 5 15:40:22 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[13/50] [abbrv] yetus git commit: HADOOP-12256. add support for
ruby-lint (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12256. add support for ruby-lint (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/fe51d665
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/fe51d665
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/fe51d665
Branch: refs/heads/master
Commit: fe51d66588e4bb91a018a4a7ee963074d280785d
Parents: 6b040e8
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 3 10:38:55 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 3 10:38:55 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch-docker/Dockerfile-startstub | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/fe51d665/dev-support/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/Dockerfile-startstub b/dev-support/test-patch-docker/Dockerfile-startstub
index 080f86f..fd3e4c5 100644
--- a/dev-support/test-patch-docker/Dockerfile-startstub
+++ b/dev-support/test-patch-docker/Dockerfile-startstub
@@ -78,6 +78,11 @@ RUN cabal update && cabal install shellcheck --global
###
RUN gem install rubocop
+####
+# Install ruby-lint
+###
+RUN gem install ruby-lint
+
#####
# Install JIRA CLI
#####
[10/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/2f353a0a
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/2f353a0a
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/2f353a0a
Branch: refs/heads/master
Commit: 2f353a0a56cdfbe2bdbc655ff775f54cfe56759a
Parents: aa21e6d c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 3 10:33:18 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 3 10:33:18 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[29/50] [abbrv] yetus git commit: HADOOP-12316. Potential
false-positive and false-negative in parsing TAP output (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12316. Potential false-positive and false-negative in parsing TAP output (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/b1c072e4
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/b1c072e4
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/b1c072e4
Branch: refs/heads/master
Commit: b1c072e45b3cd91b55d04c6874de5eba4e6553ca
Parents: 5821d5c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 13 13:35:37 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 13 13:35:37 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/tap.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/b1c072e4/dev-support/test-patch.d/tap.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/tap.sh b/dev-support/test-patch.d/tap.sh
index c6796a8..f94a14f 100755
--- a/dev-support/test-patch.d/tap.sh
+++ b/dev-support/test-patch.d/tap.sh
@@ -49,7 +49,7 @@ function tap_process_tests
declare module_failed_tests
declare filenames
- filenames=$(find "${TAP_LOG_DIR}" -type f -exec "${GREP}" -l -E "not ok " {} \;)
+ filenames=$(find "${TAP_LOG_DIR}" -type f -exec "${GREP}" -l -E "^not ok" {} \;)
if [[ -n "${filenames}" ]]; then
module_failed_tests=$(echo "${filenames}" \
@@ -73,7 +73,7 @@ function tap_finalize_results
if [[ -n "${TAP_FAILED_TESTS}" ]] ; then
# shellcheck disable=SC2086
- populate_test_table "${jdk}Failed junit tests" ${TAP_FAILED_TESTS}
+ populate_test_table "${jdk}Failed TAP tests" ${TAP_FAILED_TESTS}
TAP_FAILED_TESTS=""
add_footer_table "TAP logs" "${TAP_LOGS}"
fi
[44/50] [abbrv] yetus git commit: HADOOP-12298. releasedocmaker isn't
translating greater than/less than signs in releasenotes (aw)
Posted by bu...@apache.org.
HADOOP-12298. releasedocmaker isn't translating greater than/less than signs in releasenotes (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/e54a49b0
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/e54a49b0
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/e54a49b0
Branch: refs/heads/master
Commit: e54a49b0e7b01bfe2512ab2c790326cb995d8fca
Parents: 6f0772d
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Sep 9 08:35:24 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Sep 9 08:35:24 2015 -0700
----------------------------------------------------------------------
dev-support/docs/releasedocmaker.md | 6 ++++++
dev-support/releasedocmaker.py | 14 +++++++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/e54a49b0/dev-support/docs/releasedocmaker.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/releasedocmaker.md b/dev-support/docs/releasedocmaker.md
index b39de2e..31eb8e0 100644
--- a/dev-support/docs/releasedocmaker.md
+++ b/dev-support/docs/releasedocmaker.md
@@ -61,6 +61,12 @@ $ releasedocmaker.py --project HBASE --version 1.2.0
... will create a 1.2.0 directory and inside that directory will be CHANGES.1.2.0.md and RELEASENOTES.1.2.0.md .
+By default, release notes are expected to be in plain text. However, you can write them in markdown if you include a header at the top of your release note:
+
+```xml
+<!-- markdown -->
+remaining text
+```
# Changing the Header
http://git-wip-us.apache.org/repos/asf/yetus/blob/e54a49b0/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 3c398be..d62c0d1 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -31,6 +31,7 @@ except ImportError:
RELEASE_VERSION = {}
NAME_PATTERN = re.compile(r' \([0-9]+\)')
+RELNOTE_PATTERN = re.compile('^\<\!\-\- ([a-z]+) \-\-\>')
ASF_LICENSE = '''
<!---
@@ -88,6 +89,17 @@ def notableclean(_str):
_str = _str.rstrip()
return _str
+# if release notes have a special marker,
+# we'll treat them as already in markdown format
+def processrelnote(_str):
+ fmt = RELNOTE_PATTERN.match(_str)
+ if fmt is None:
+ return notableclean(_str)
+ else:
+ return {
+ 'markdown' : tableclean(_str),
+ }.get(fmt.group(1),notableclean(_str))
+
# clean output dir
def clean_output_dir(directory):
files = os.listdir(directory)
@@ -519,7 +531,7 @@ def main():
if len(jira.get_release_note()) > 0:
reloutputs.write_key_raw(jira.get_project(), "\n---\n\n")
reloutputs.write_key_raw(jira.get_project(), line)
- line = '\n%s\n\n' % (tableclean(jira.get_release_note()))
+ line = '\n%s\n\n' % (processrelnote(jira.get_release_note()))
reloutputs.write_key_raw(jira.get_project(), line)
if options.lint is True:
[11/50] [abbrv] yetus git commit: HADOOP-12286. test-patch pylint
plugin should support indent-string option (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12286. test-patch pylint plugin should support indent-string option (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/20b7d819
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/20b7d819
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/20b7d819
Branch: refs/heads/master
Commit: 20b7d8193f2465c292132d8aba28eb42d680092a
Parents: 2f353a0
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 3 10:34:01 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 3 10:34:01 2015 -0700
----------------------------------------------------------------------
dev-support/personality/hadoop.sh | 2 ++
dev-support/test-patch.d/pylint.sh | 19 +++++++++++++------
2 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/20b7d819/dev-support/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hadoop.sh b/dev-support/personality/hadoop.sh
index 3d6e3fa..60dbb3d 100755
--- a/dev-support/personality/hadoop.sh
+++ b/dev-support/personality/hadoop.sh
@@ -22,6 +22,8 @@ PATCH_BRANCH_DEFAULT=trunk
HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
#shellcheck disable=SC2034
ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
+#shellcheck disable=SC2034
+PYLINT_OPTIONS="--indent-string=' '"
HADOOP_MODULES=""
http://git-wip-us.apache.org/repos/asf/yetus/blob/20b7d819/dev-support/test-patch.d/pylint.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/pylint.sh b/dev-support/test-patch.d/pylint.sh
index faa8136..ebac162 100755
--- a/dev-support/test-patch.d/pylint.sh
+++ b/dev-support/test-patch.d/pylint.sh
@@ -19,11 +19,13 @@ add_plugin pylint
PYLINT_TIMER=0
PYLINT=${PYLINT:-$(which pylint 2>/dev/null)}
+PYLINT_OPTIONS=${PYLINT_OPTIONS:-}
function pylint_usage
{
echo "Pylint specific:"
- echo "--pylint=<path> path to pylint executable"
+ echo "--pylint=<path> path to pylint executable"
+ echo "--pylint-options=<path> pylint options other than output-format and reports"
}
function pylint_parse_args
@@ -35,6 +37,9 @@ function pylint_parse_args
--pylint=*)
PYLINT=${i#*=}
;;
+ --pylint-options=*)
+ PYLINT_OPTIONS=${i#*=}
+ ;;
esac
done
}
@@ -70,7 +75,8 @@ function pylint_preapply
pushd "${BASEDIR}" >/dev/null
for i in ${CHANGED_FILES}; do
if [[ ${i} =~ \.py$ && -f ${i} ]]; then
- ${PYLINT} --indent-string=" " --output-format=parseable --reports=n "${i}" 2>/dev/null |
+ # shellcheck disable=SC2086
+ eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>/dev/null |
${AWK} '1<NR' >> "${PATCH_DIR}/branch-pylint-result.txt"
fi
done
@@ -111,7 +117,8 @@ function pylint_postapply
pushd "${BASEDIR}" >/dev/null
for i in ${CHANGED_FILES}; do
if [[ ${i} =~ \.py$ && -f ${i} ]]; then
- ${PYLINT} --indent-string=" " --output-format=parseable --reports=n "${i}" 2>/dev/null |
+ # shellcheck disable=SC2086
+ eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>/dev/null |
${AWK} '1<NR' >> "${PATCH_DIR}/patch-pylint-result.txt"
fi
done
@@ -122,14 +129,14 @@ function pylint_postapply
add_footer_table pylint "v${PYLINT_VERSION%,}"
calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" "${PATCH_DIR}/patch-pylint-result.txt" > "${PATCH_DIR}/diff-patch-pylint.txt"
- diffPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-pylint.txt")
+ diffPostpatch=$(${AWK} 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-pylint.txt")
if [[ ${diffPostpatch} -gt 0 ]] ; then
# shellcheck disable=SC2016
- numPrepatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-pylint-result.txt")
+ numPrepatch=$(${AWK} 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-pylint-result.txt")
# shellcheck disable=SC2016
- numPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-pylint-result.txt")
+ numPostpatch=$(${AWK} 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-pylint-result.txt")
add_vote_table -1 pylint "The applied patch generated "\
"${diffPostpatch} new pylint issues (total was ${numPrepatch}, now ${numPostpatch})."
[08/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/a1b7bb3b
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/a1b7bb3b
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/a1b7bb3b
Branch: refs/heads/master
Commit: a1b7bb3b13942578a0d3d4845acfe4a6fa6eaa71
Parents: 84fea27 c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jul 31 15:21:35 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jul 31 15:21:35 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[34/50] [abbrv] yetus git commit: HADOOP-12340. test-patch docker
mode fails in downloading findbugs with curl (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12340. test-patch docker mode fails in downloading findbugs with curl (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/2340c552
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/2340c552
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/2340c552
Branch: refs/heads/master
Commit: 2340c552117ae4d369bb291768f410bcfea8b421
Parents: f3562c6
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 25 09:12:47 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Aug 25 09:12:47 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch-docker/Dockerfile-startstub | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/2340c552/dev-support/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/Dockerfile-startstub b/dev-support/test-patch-docker/Dockerfile-startstub
index c49b589..560d751 100644
--- a/dev-support/test-patch-docker/Dockerfile-startstub
+++ b/dev-support/test-patch-docker/Dockerfile-startstub
@@ -62,7 +62,7 @@ RUN apt-get install -y oracle-java8-installer
# Install findbugs
######
RUN mkdir -p /opt/findbugs && \
- curl https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
+ curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
-o /opt/findbugs.tar.gz && \
tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
ENV FINDBUGS_HOME /opt/findbugs
[41/50] [abbrv] yetus git commit: HADOOP-12355. test-patch TAP plugin
should use ${SED} instead of sed (Jagadesh Kiran N via aw)
Posted by bu...@apache.org.
HADOOP-12355. test-patch TAP plugin should use ${SED} instead of sed (Jagadesh Kiran N via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/c7022333
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/c7022333
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/c7022333
Branch: refs/heads/master
Commit: c7022333e216911f1b0dfd896383370111afd74a
Parents: b05d4c3
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 4 14:45:29 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 4 14:45:29 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/tap.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/c7022333/dev-support/test-patch.d/tap.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/tap.sh b/dev-support/test-patch.d/tap.sh
index f94a14f..7ced908 100755
--- a/dev-support/test-patch.d/tap.sh
+++ b/dev-support/test-patch.d/tap.sh
@@ -53,7 +53,7 @@ function tap_process_tests
if [[ -n "${filenames}" ]]; then
module_failed_tests=$(echo "${filenames}" \
- | sed -e "s,${TAP_LOG_DIR},,g" -e s,^/,,g )
+ | ${SED} -e "s,${TAP_LOG_DIR},,g" -e s,^/,,g )
# shellcheck disable=SC2086
cat ${filenames} >> "${PATCH_DIR}/patch-${filefrag}.tap"
TAP_LOGS="${TAP_LOGS} @@BASE@@/patch-${filefrag}.tap"
[40/50] [abbrv] yetus git commit: HADOOP-12349. Misleading debug
message in generic_locate_patch (Jagadesh Kiran N via aw)
Posted by bu...@apache.org.
HADOOP-12349. Misleading debug message in generic_locate_patch (Jagadesh Kiran N via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/b05d4c31
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/b05d4c31
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/b05d4c31
Branch: refs/heads/master
Commit: b05d4c313f534d08ce58d496214fde2002233c44
Parents: b8a06f9
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 4 14:44:02 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 4 14:44:16 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/builtin-bugsystem.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/b05d4c31/dev-support/test-patch.d/builtin-bugsystem.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/builtin-bugsystem.sh b/dev-support/test-patch.d/builtin-bugsystem.sh
index 9a9ee05..de0f01b 100644
--- a/dev-support/test-patch.d/builtin-bugsystem.sh
+++ b/dev-support/test-patch.d/builtin-bugsystem.sh
@@ -35,7 +35,7 @@ function generic_locate_patch
--output "${output}" \
"${input}"
if [[ $? != 0 ]]; then
- yetus_debug "jira_locate_patch: not a JIRA."
+ yetus_debug "generic_locate_patch: failed to download the patch."
return 1
fi
return 0
[39/50] [abbrv] yetus git commit: HADOOP-12312. Findbugs HTML report
link shows 0 warnings despite errors (Kengo Seki via busbey)
Posted by bu...@apache.org.
HADOOP-12312. Findbugs HTML report link shows 0 warnings despite errors (Kengo Seki via busbey)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/b8a06f99
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/b8a06f99
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/b8a06f99
Branch: refs/heads/master
Commit: b8a06f99262b6dee0abbb765e30bc2bbaa021a0f
Parents: 30d8e14
Author: Sean Busbey <bu...@apache.org>
Authored: Fri Sep 4 09:16:42 2015 -0500
Committer: Sean Busbey <bu...@cloudera.com>
Committed: Fri Sep 4 09:17:09 2015 -0500
----------------------------------------------------------------------
dev-support/test-patch.d/findbugs.sh | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/b8a06f99/dev-support/test-patch.d/findbugs.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/findbugs.sh b/dev-support/test-patch.d/findbugs.sh
index 4fa5428..16ef8d7 100755
--- a/dev-support/test-patch.d/findbugs.sh
+++ b/dev-support/test-patch.d/findbugs.sh
@@ -252,7 +252,9 @@ function findbugs_postinstall
local branchxml
local patchxml
local newbugsbase
+ local fixedbugsbase
local new_findbugs_warnings
+ local fixed_findbugs_warnings
local line
local firstpart
local secondpart
@@ -297,6 +299,7 @@ function findbugs_postinstall
fi
newbugsbase="${PATCH_DIR}/new-findbugs-${fn}"
+ fixedbugsbase="${PATCH_DIR}/fixed-findbugs-${fn}"
"${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
-output "${combined_xml}" \
@@ -326,8 +329,8 @@ function findbugs_postinstall
fi
#shellcheck disable=SC2016
- new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
- "${combined_xml}" "${newbugsbase}.xml" | ${AWK} '{print $1}')
+ fixed_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
+ "${combined_xml}" "${fixedbugsbase}.xml" | ${AWK} '{print $1}')
if [[ $? != 0 ]]; then
popd >/dev/null
module_status ${i} -1 "" "${module} cannot run filterBugs (#2) from findbugs"
@@ -338,9 +341,7 @@ function findbugs_postinstall
continue
fi
- echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
- findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
- findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
+ echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${fixed_findbugs_warnings} newly fixed warnings."
"${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newbugsbase}.xml" \
"${newbugsbase}.html"
[38/50] [abbrv] yetus git commit: HADOOP-12336. github_jira_bridge
doesn't work (Kengo Seki)
Posted by bu...@apache.org.
HADOOP-12336. github_jira_bridge doesn't work (Kengo Seki)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/30d8e147
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/30d8e147
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/30d8e147
Branch: refs/heads/master
Commit: 30d8e1473ee412977d2a15a2fe7e5c63599a05b1
Parents: 3ee8491
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Sep 2 18:06:10 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Sep 2 18:06:10 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/github.sh | 7 ++-----
dev-support/test-patch.d/jira.sh | 8 +++++++-
2 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/30d8e147/dev-support/test-patch.d/github.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/github.sh b/dev-support/test-patch.d/github.sh
index 36c7e51..f742062 100755
--- a/dev-support/test-patch.d/github.sh
+++ b/dev-support/test-patch.d/github.sh
@@ -89,12 +89,9 @@ function github_jira_bridge
# we use this to prevent loops later on
GITHUB_BRIDGED=true
- # the JIRA issue has already been downloaded. So let's
- # find the URL. This is currently hard-coded to github.com
- # Sorry Github Enterprise users. :(
-
+ # the JIRA issue has already been downloaded. So let's find the URL.
# shellcheck disable=SC2016
- urlfromjira=$(${AWK} 'match($0,"https://github.com/.*patch"){print $1}' "${PATCH_DIR}/jira" | tail -1)
+ urlfromjira=$(${AWK} "match(\$0,\"${GITHUB_BASE_URL}/[^ ]*patch\"){print substr(\$0,RSTART,RLENGTH)}" "${PATCH_DIR}/jira" | tail -1)
github_breakup_url "${urlfromjira}"
github_locate_patch "${GITHUB_ISSUE}" "${fileloc}"
}
http://git-wip-us.apache.org/repos/asf/yetus/blob/30d8e147/dev-support/test-patch.d/jira.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/jira.sh b/dev-support/test-patch.d/jira.sh
index ca9f2bd..fc649e7 100755
--- a/dev-support/test-patch.d/jira.sh
+++ b/dev-support/test-patch.d/jira.sh
@@ -64,15 +64,20 @@ function jira_determine_issue
declare patchnamechunk
declare maybeissue
+ if [[ -n ${JIRA_ISSUE} ]]; then
+ return 0
+ fi
+
# shellcheck disable=SC2016
patchnamechunk=$(echo "${input}" | ${AWK} -F/ '{print $NF}')
maybeissue=$(echo "${patchnamechunk}" | cut -f1,2 -d-)
if [[ ${maybeissue} =~ ${JIRA_ISSUE_RE} ]]; then
+ # shellcheck disable=SC2034
ISSUE=${maybeissue}
JIRA_ISSUE=${maybeissue}
- add_footer_table "JIRA Issue" "${ISSUE}"
+ add_footer_table "JIRA Issue" "${JIRA_ISSUE}"
return 0
fi
@@ -123,6 +128,7 @@ function jira_locate_patch
# send this to the github plugin to process.
if [[ -n "${GITHUB_BASE_URL}"
&& $(${GREP} -c "${GITHUB_BASE_URL}"'[^ ]*patch' "${PATCH_DIR}/jira") != 0 ]]; then
+ jira_determine_issue "${input}"
echo "${input} appears to be a Github PR. Switching Modes."
github_jira_bridge "${fileloc}"
return $?
[09/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/aa21e6d4
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/aa21e6d4
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/aa21e6d4
Branch: refs/heads/master
Commit: aa21e6d4990a4d8a033b108ecbd2b90d32ae818e
Parents: a1b7bb3 c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jul 31 23:15:28 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jul 31 23:15:28 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[12/50] [abbrv] yetus git commit: HADOOP-12287. add support for
perlcritic (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12287. add support for perlcritic (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/6b040e80
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/6b040e80
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/6b040e80
Branch: refs/heads/master
Commit: 6b040e8054e2da55f087e957035a46e7fc30657e
Parents: 20b7d81
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 3 10:36:24 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 3 10:36:24 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch-docker/Dockerfile-startstub | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/6b040e80/dev-support/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/Dockerfile-startstub b/dev-support/test-patch-docker/Dockerfile-startstub
index 5e5ca78..080f86f 100644
--- a/dev-support/test-patch-docker/Dockerfile-startstub
+++ b/dev-support/test-patch-docker/Dockerfile-startstub
@@ -34,7 +34,8 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
libcurl4-openssl-dev \
python python2.7 pylint \
ruby \
- openjdk-7-jdk
+ openjdk-7-jdk \
+ libperl-critic-perl
# Fixing the Apache commons / Maven dependency problem under Ubuntu:
# See http://wiki.apache.org/commons/VfsProblems
[23/50] [abbrv] yetus git commit: HADOOP-12248. Add native support
for TAP (aw)
Posted by bu...@apache.org.
HADOOP-12248. Add native support for TAP (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/91cbda73
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/91cbda73
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/91cbda73
Branch: refs/heads/master
Commit: 91cbda73631844e759fd3634d19441371af617fd
Parents: e207ab5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 11 07:49:15 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Aug 11 07:49:15 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/tap.sh | 80 ++++++++++++++++++++++++++++++++++++
1 file changed, 80 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/91cbda73/dev-support/test-patch.d/tap.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/tap.sh b/dev-support/test-patch.d/tap.sh
new file mode 100755
index 0000000..c6796a8
--- /dev/null
+++ b/dev-support/test-patch.d/tap.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_test_format tap
+
+TAP_FAILED_TESTS=""
+TAP_LOG_DIR="target/tap"
+
+function tap_process_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --tap-log-dir=*)
+ TAP_LOG_DIR=${i#=*}
+ ;;
+ esac
+ done
+}
+
+function tap_usage
+{
+ echo "TAP Options:"
+ echo "--tap-log-dir=<dir> Directory relative to the module for tap output (default: \"target/tap\")"
+}
+
+function tap_process_tests
+{
+ # shellcheck disable=SC2034
+ declare module=$1
+ # shellcheck disable=SC2034
+ declare buildlogfile=$2
+ declare filefrag=$3
+ declare result=0
+ declare module_failed_tests
+ declare filenames
+
+ filenames=$(find "${TAP_LOG_DIR}" -type f -exec "${GREP}" -l -E "not ok " {} \;)
+
+ if [[ -n "${filenames}" ]]; then
+ module_failed_tests=$(echo "${filenames}" \
+ | sed -e "s,${TAP_LOG_DIR},,g" -e s,^/,,g )
+ # shellcheck disable=SC2086
+ cat ${filenames} >> "${PATCH_DIR}/patch-${filefrag}.tap"
+ TAP_LOGS="${TAP_LOGS} @@BASE@@/patch-${filefrag}.tap"
+ TAP_FAILED_TESTS="${TAP_FAILED_TESTS} ${module_failed_tests}"
+ ((result=result+1))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+function tap_finalize_results
+{
+ declare jdk=$1
+
+ if [[ -n "${TAP_FAILED_TESTS}" ]] ; then
+ # shellcheck disable=SC2086
+ populate_test_table "${jdk}Failed junit tests" ${TAP_FAILED_TESTS}
+ TAP_FAILED_TESTS=""
+ add_footer_table "TAP logs" "${TAP_LOGS}"
+ fi
+}
[02/50] [abbrv] yetus git commit: HADOOP-12228. Document
releasedocmaker (aw)
Posted by bu...@apache.org.
HADOOP-12228. Document releasedocmaker (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/39aebf60
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/39aebf60
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/39aebf60
Branch: refs/heads/master
Commit: 39aebf60fbf19e02d3ed34cfab4ed6c7113bbb76
Parents: 91e48da
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Jul 30 17:06:02 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Jul 30 17:06:02 2015 -0700
----------------------------------------------------------------------
dev-support/docs/releasedocmaker.md | 230 +++++++++++++++++++++++++++++++
1 file changed, 230 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/39aebf60/dev-support/docs/releasedocmaker.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/releasedocmaker.md b/dev-support/docs/releasedocmaker.md
new file mode 100644
index 0000000..d8eb13c
--- /dev/null
+++ b/dev-support/docs/releasedocmaker.md
@@ -0,0 +1,230 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+releasedocmaker
+===============
+
+* [Purpose](#Purpose)
+* [Basic Usage](#Basic_Usage)
+* [Changing the Header](#Changing_the_Header)
+* [Multiple Versions](#Multiple_Versions)
+* [Unreleased Dates](#Unreleased_Dates)
+* [Lint Mode](#Lint_Mode)
+
+# Purpose
+
+Building changelog information in a form that is human digestible but still containing as much useful information is difficult. Many attempts over the years have resulted in a variety of methods that projects use to solve this problem:
+
+* JIRA-generated release notes from the "Release Notes" button
+* Manually modified CHANGES file
+* Processing git log information
+
+All of these methods have their pros and cons. Some have issues with accuracy. Some have issues with lack of details. None of these methods seem to cover all of the needs of many projects and are full of potential pitfalls.
+
+In order to solve these problems, releasedocmaker was written to automatically generate a changelog and release notes by querying Apache's JIRA instance.
+
+# Basic Usage
+
+Minimally, the name of the JIRA project and a version registered in JIRA must be provided:
+
+```bash
+$ releasedocmaker.py --project (project) --version (version)
+```
+
+This will query Apache JIRA, generating two files in a directory named after the given version in an extended markdown format which can be processed by both mvn site and GitHub.
+
+* CHANGES.(version).md
+
+This is similar to the JIRA "Release Notes" button but is in tabular format and includes the priority, component, reporter, and contributor fields. It also highlights Incompatible Changes so that readers know what to look out for when upgrading. The top of the file also includes the date that the version was marked as released in JIRA.
+
+
+* RELEASENOTES.(version).md
+
+If your JIRA project supports the release note field, this will contain any JIRA mentioned in the CHANGES log that is either an incompatible change or has a release note associated with it. If your JIRA project does not support the release notes field, this will be the description field.
+
+For example, to build the release documentation for HBase v1.2.0...
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.2.0
+```
+
+... will create a 1.2.0 directory and inside that directory will be CHANGES.1.2.0.md and RELEASENOTES.1.2.0.md .
+
+
+# Changing the Header
+
+By default, it will use a header that matches the project name. But that is kind of ugly and the case may be wrong. Luckily, the title can be changed:
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.2.0 --projecttitle "Apache HBase"
+```
+
+Now instead of "HBASE", it will use "Apache HBASE" for some titles and headers.
+
+# Multiple Versions
+
+The script can also generate multiple versions at once, by
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --version 1.2.0
+```
+
+This will create the files for versions 1.0.0 and versions 1.2.0 in their own directories.
+
+But what if the version numbers are not known? releasedocmaker can also generate version data based upon ranges:
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --version 1.2.0 --range
+```
+
+In this form, releasedocmaker will query JIRA, discover all versions that alphabetically appear to be between 1.0.0 and 1.2.0, inclusive, and generate all of the relative release documents. This is especially useful when bootstrapping an existing project.
+
+# Unreleased Dates
+
+For released versions, releasedocmaker will pull the date of the release from JIRA. However, for unreleased versions it marks the release as "Unreleased". This can be inconvenient when actually building a release and wanting to include it inside the source package.
+
+The --usetoday option can be used to signify that instead of using Unreleased, releasedocmaker should use today's date.
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --usetoday
+```
+
+After using this option and release, don't forget to change JIRA's release date to match!
+
+# Lint Mode
+
+In order to ensure proper formatting while using mvn site, releasedocmaker puts in periods (.) for fields that are empty or unassigned. This can be unsightly and not proper for any given project. There are also other things, such as missing release notes for incompatible changes, that are less than desirable.
+
+In order to help release managers from having to scan through potentially large documents, releasedocmaker features a lint mode, triggered via --lint:
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --lint
+```
+
+This will do the normal JIRA querying, looking for items it considers problematic. It will print the information to the screen and then exit with either success or failure, depending upon if any issues were discovered.
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+releasedocmaker
+===============
+
+* [Purpose](#Purpose)
+* [Basic Usage](#Basic_Usage)
+* [Changing the Header](#Changing_the_Header)
+* [Multiple Versions](#Multiple_Versions)
+* [Unreleased Dates](#Unreleased_Dates)
+* [Lint Mode](#Lint_Mode)
+
+# Purpose
+
+Building changelog information in a form that is human digestible but still containing as much useful information is difficult. Many attempts over the years have resulted in a variety of methods that projects use to solve this problem:
+
+* JIRA-generated release notes from the "Release Notes" button
+* Manually modified CHANGES file
+* Processing git log information
+
+All of these methods have their pros and cons. Some have issues with accuracy. Some have issues with lack of details. None of these methods seem to cover all of the needs of many projects and are full of potential pitfalls.
+
+In order to solve these problems, releasedocmaker was written to automatically generate a changelog and release notes by querying Apache's JIRA instance.
+
+# Basic Usage
+
+Minimally, the name of the JIRA project and a version registered in JIRA must be provided:
+
+```bash
+$ releasedocmaker.py --project (project) --version (version)
+```
+
+This will query Apache JIRA, generating two files in a directory named after the given version in an extended markdown format which can be processed by both mvn site and GitHub.
+
+* CHANGES.(version).md
+
+This is similar to the JIRA "Release Notes" button but is in tabular format and includes the priority, component, reporter, and contributor fields. It also highlights Incompatible Changes so that readers know what to look out for when upgrading. The top of the file also includes the date that the version was marked as released in JIRA.
+
+
+* RELEASENOTES.(version).md
+
+If your JIRA project supports the release note field, this will contain any JIRA mentioned in the CHANGES log that is either an incompatible change or has a release note associated with it. If your JIRA project does not support the release notes field, this will be the description field.
+
+For example, to build the release documentation for HBase v1.2.0...
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.2.0
+```
+
+... will create a 1.2.0 directory and inside that directory will be CHANGES.1.2.0.md and RELEASENOTES.1.2.0.md .
+
+
+# Changing the Header
+
+By default, it will use a header that matches the project name. But that is kind of ugly and the case may be wrong. Luckily, the title can be changed:
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.2.0 --projecttitle "Apache HBase"
+```
+
+Now instead of "HBASE", it will use "Apache HBASE" for some titles and headers.
+
+# Multiple Versions
+
+The script can also generate multiple versions at once, by
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --version 1.2.0
+```
+
+This will create the files for versions 1.0.0 and versions 1.2.0 in their own directories.
+
+But what if the version numbers are not known? releasedocmaker can also generate version data based upon ranges:
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --version 1.2.0 --range
+```
+
+In this form, releasedocmaker will query JIRA, discover all versions that alphabetically appear to be between 1.0.0 and 1.2.0, inclusive, and generate all of the relative release documents. This is especially useful when bootstrapping an existing project.
+
+# Unreleased Dates
+
+For released versions, releasedocmaker will pull the date of the release from JIRA. However, for unreleased versions it marks the release as "Unreleased". This can be inconvenient when actually building a release and wanting to include it inside the source package.
+
+The --usetoday option can be used to signify that instead of using Unreleased, releasedocmaker should use today's date.
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --usetoday
+```
+
+After using this option and release, don't forget to change JIRA's release date to match!
+
+# Lint Mode
+
+In order to ensure proper formatting while using mvn site, releasedocmaker puts in periods (.) for fields that are empty or unassigned. This can be unsightly and not proper for any given project. There are also other things, such as missing release notes for incompatible changes, that are less than desirable.
+
+In order to help release managers from having to scan through potentially large documents, releasedocmaker features a lint mode, triggered via --lint:
+
+```bash
+$ releasedocmaker.py --project HBASE --version 1.0.0 --lint
+```
+
+This will do the normal JIRA querying, looking for items it considers problematic. It will print the information to the screen and then exit with either success or failure, depending upon if any issues were discovered.
[27/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Conflicts:
dev-support/test-patch.sh
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/942def2d
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/942def2d
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/942def2d
Branch: refs/heads/master
Commit: 942def2dc5e28b9b4c434a1e7763386e153654bc
Parents: d32053a 8369b1c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 13 12:52:23 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 13 12:52:23 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[33/50] [abbrv] yetus git commit: HADOOP-12303. test-patch pylint
plugin fails silently and votes +1 incorrectly (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12303. test-patch pylint plugin fails silently and votes +1 incorrectly (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/f3562c67
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/f3562c67
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/f3562c67
Branch: refs/heads/master
Commit: f3562c67f358787cae0271c16bc0485eb69abea1
Parents: 42fd07c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 20 08:19:15 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 20 08:19:41 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/pylint.sh | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/f3562c67/dev-support/test-patch.d/pylint.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/pylint.sh b/dev-support/test-patch.d/pylint.sh
index ebac162..1ee5a8f 100755
--- a/dev-support/test-patch.d/pylint.sh
+++ b/dev-support/test-patch.d/pylint.sh
@@ -56,6 +56,8 @@ function pylint_filefilter
function pylint_preapply
{
local i
+ local count
+ local tmp=${PATCH_DIR}/pylint.$$.${RANDOM}
verify_needed_test pylint
if [[ $? == 0 ]]; then
@@ -76,10 +78,17 @@ function pylint_preapply
for i in ${CHANGED_FILES}; do
if [[ ${i} =~ \.py$ && -f ${i} ]]; then
# shellcheck disable=SC2086
- eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>/dev/null |
+ eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>${tmp} |
${AWK} '1<NR' >> "${PATCH_DIR}/branch-pylint-result.txt"
fi
+ # shellcheck disable=SC2016
+ count=$(${GREP} -v "^No config file found" "${tmp}" | wc -l | ${AWK} '{print $1}')
+ if [[ ${count} -gt 0 ]]; then
+ add_footer_table pylint "prepatch stderr: ${tmp}"
+ return 1
+ fi
done
+ rm "${tmp}" 2>/dev/null
popd >/dev/null
# keep track of how much as elapsed for us already
PYLINT_TIMER=$(stop_clock)
@@ -89,9 +98,11 @@ function pylint_preapply
function pylint_postapply
{
local i
+ local count
local numPrepatch
local numPostpatch
local diffPostpatch
+ local tmp=${PATCH_DIR}/pylint.$$.${RANDOM}
verify_needed_test pylint
if [[ $? == 0 ]]; then
@@ -118,10 +129,18 @@ function pylint_postapply
for i in ${CHANGED_FILES}; do
if [[ ${i} =~ \.py$ && -f ${i} ]]; then
# shellcheck disable=SC2086
- eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>/dev/null |
+ eval "${PYLINT} ${PYLINT_OPTIONS} --output-format=parseable --reports=n ${i}" 2>${tmp} |
${AWK} '1<NR' >> "${PATCH_DIR}/patch-pylint-result.txt"
fi
+ # shellcheck disable=SC2016
+ count=$(${GREP} -v "^No config file found" "${tmp}" | wc -l | ${AWK} '{print $1}')
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table -1 pylint "Something bad seems to have happened in running pylint. Please check pylint stderr files."
+ add_footer_table pylint "postpatch stderr: ${tmp}"
+ return 1
+ fi
done
+ rm "${tmp}" 2>/dev/null
popd >/dev/null
# shellcheck disable=SC2016
[49/50] [abbrv] yetus git commit: HADOOP-12398. filefilter function
in test-patch flink personality is never called (Jagadesh Kiran N via aw)
Posted by bu...@apache.org.
HADOOP-12398. filefilter function in test-patch flink personality is never called (Jagadesh Kiran N via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/1c4921c3
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/1c4921c3
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/1c4921c3
Branch: refs/heads/master
Commit: 1c4921c34efdd58ce25c9bbe0a84fc8b2eb2acad
Parents: 6178096
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 11 17:57:46 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 11 17:57:46 2015 -0700
----------------------------------------------------------------------
dev-support/personality/flink.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/1c4921c3/dev-support/personality/flink.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/flink.sh b/dev-support/personality/flink.sh
index fbf0ba0..9ee11f3 100755
--- a/dev-support/personality/flink.sh
+++ b/dev-support/personality/flink.sh
@@ -25,7 +25,7 @@ HOW_TO_CONTRIBUTE=""
add_plugin flinklib
-function fliblib_filefilter
+function flinklib_filefilter
{
local filename=$1
[30/50] [abbrv] yetus git commit: HADOOP-12314. check_unittests in
test-patch.sh can return a wrong status (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12314. check_unittests in test-patch.sh can return a wrong status (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/0d65ccd0
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/0d65ccd0
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/0d65ccd0
Branch: refs/heads/master
Commit: 0d65ccd03413307a79c306639bbde5e9678aa61d
Parents: b1c072e
Author: Allen Wittenauer <aw...@apache.org>
Authored: Sat Aug 15 11:12:51 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Sat Aug 15 11:12:51 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/0d65ccd0/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 13332b7..a368f83 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -2118,6 +2118,7 @@ function modules_workers
local jdk=""
local jdkindex=0
local statusjdk
+ local result=0
if [[ ${repostatus} == branch ]]; then
repo=${PATCH_BRANCH}
@@ -2834,11 +2835,11 @@ function precheck_mvninstall
big_console_header "Verifying mvn install works"
verify_needed_test javadoc
- retval=$?
+ result=$?
verify_needed_test javac
- ((retval = retval + $? ))
- if [[ ${retval} == 0 ]]; then
+ ((result = result + $? ))
+ if [[ ${result} == 0 ]]; then
echo "This patch does not appear to need mvn install checks."
return 0
fi
@@ -2870,11 +2871,11 @@ function check_mvninstall
big_console_header "Verifying mvn install still works"
verify_needed_test javadoc
- retval=$?
+ result=$?
verify_needed_test javac
- ((retval = retval + $? ))
- if [[ ${retval} == 0 ]]; then
+ ((result = result + $? ))
+ if [[ ${result} == 0 ]]; then
echo "This patch does not appear to need mvn install checks."
return 0
fi
@@ -2897,6 +2898,8 @@ function check_mvninstall
## @return 1 on failure
function check_mvn_eclipse
{
+ local result=0
+
if [[ ${BUILDTOOL} != maven ]]; then
return 0
fi
@@ -3021,7 +3024,7 @@ function check_unittests
yetus_debug "Calling ${testsys}_process_tests"
"${testsys}_process_tests" "${module}" "${test_logfile}" "${fn}"
formatresult=$?
- ((results=results+formatresult))
+ ((result=result+formatresult))
if [[ "${formatresult}" != 0 ]]; then
needlog=1
fi
[07/50] [abbrv] yetus git commit: HADOOP-12130. document features
added in 12113 (aw)
Posted by bu...@apache.org.
HADOOP-12130. document features added in 12113 (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/84fea273
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/84fea273
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/84fea273
Branch: refs/heads/master
Commit: 84fea273df6f9268a3d6c3a36a7c44c82fa79702
Parents: d2114ff
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jul 31 14:53:29 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jul 31 14:53:29 2015 -0700
----------------------------------------------------------------------
dev-support/docs/precommit-advanced.md | 68 ++++++++++++++----
dev-support/docs/precommit-architecture.md | 31 ++++----
dev-support/docs/precommit-basic.md | 94 +++++++++++++++++++++----
3 files changed, 149 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/84fea273/dev-support/docs/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-advanced.md b/dev-support/docs/precommit-advanced.md
index 0a7eac5..a424199 100644
--- a/dev-support/docs/precommit-advanced.md
+++ b/dev-support/docs/precommit-advanced.md
@@ -16,22 +16,35 @@ test-patch
==========
* [Docker Support](#Docker_Support)
-* [Maven Profiles](#Maven_Profiles)
+* [Maven Specific](#Maven_Specific)
+* [Ant Specific](#Ant_Specific)
* [Plug-ins](#Plug-ins)
* [Configuring for Other Projects](#Configuring_for_Other_Projects)
+* [Important Variables](#Important_Variables)
# Docker Support
-By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself into a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, the patch requires a newer version of Java.
+By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself into a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java.
-The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. Be aware that test-patch will copy this file and append its necessary hooks in order to execute.
+The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. However be aware that test-patch will copy this file and append its necessary hooks to re-launch itself prior to executing docker.
-test-patch includes code to automatically manage broken/stale container images that are hanging around if it is run in --jenkins mode. In this way, if Docker fails to build the image, the disk space should eventually return.
+Dockerfile images will be named with a test-patch prefix and suffix with either a date or a git commit hash. By using this information, test-patch will automatically manage broken/stale container images that are hanging around if it is run in --jenkins mode. In this way, if Docker fails to build the image, the disk space should eventually be cleaned and returned back to the system.
-# Maven Profiles
+# Maven Specific
-By default, test-patch will pass -Ptest-patch and -D${PROJECT_NAME}PatchProcess to Maven. This will allow you to configure special actions that should only happen when running underneath test-patch.
+## Command Arguments
+test-patch always passes --batch-mode to maven to force it into non-interactive mode. Additionally, some tests will also force -fae in order to get all of messages/errors during that mode. It *does not* pass -DskipTests. Additional arguments should be handled via the personality.
+
+## Test Profile
+
+By default, test-patch will pass -Ptest-patch to Maven. This will allow you to configure special actions that should only happen when running underneath test-patch.
+
+# Ant Specific
+
+## Command Arguments
+
+test-patch always passes -noinput to Ant. This force ant to be non-interactive.
# Plug-ins
@@ -58,7 +71,7 @@ Similarly, there are other functions that may be defined during the test-patch r
- executed prior to the patch being applied but after the git repository is setup. This is useful for any early error checking that might need to be done before any heavier work.
* pluginname_preapply
- - executed prior to the patch being applied. This is useful for any "before"-type data collection for later comparisons
+ - executed prior to the patch being applied. This is useful for any "before"-type data collection for later comparisons.
* pluginname_postapply
- executed after the patch has been applied. This is useful for any "after"-type data collection.
@@ -79,12 +92,12 @@ If the plug-in has some specific options, one can use following functions:
- executed prior to any other above functions except for pluginname_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
- HINT: It is recommend to make the pluginname relatively small, 10 characters at the most. Otherwise the ASCII output table may be skewed.
+ HINT: It is recommended to make the pluginname relatively small, 10 characters at the most. Otherwise, the ASCII output table may be skewed.
# Configuring for Other Projects
-It is impossible for any general framework to be predictive about what types of special rules any given project may have, especially when it comes to ordering and Maven profiles. In order to assist non-Hadoop projects, a project `personality` should be added that enacts these custom rules.
+It is impossible for any general framework to be predictive about what types of special rules any given project may have, especially when it comes to ordering and Maven profiles. In order to direct test-patch to do the correct action, a project `personality` should be added that enacts these custom rules.
A personality consists of two functions. One that determines which test types to run and another that allows a project to dictate ordering rules, flags, and profiles on a per-module, per-test run.
@@ -92,7 +105,7 @@ There can be only **one** of each personality function defined.
## Test Determination
-The `personality_file_tests` function determines which tests to turn on based upon the file name. It is realtively simple. For example, to turn on a full suite of tests for Java files:
+The `personality_file_tests` function determines which tests to turn on based upon the file name. It is relatively simple. For example, to turn on a full suite of tests for Java files:
```bash
function personality_file_tests
@@ -131,19 +144,19 @@ function personality_modules
It takes exactly two parameters `repostatus` and `testtype`.
-The `repostatus` parameter tells the `personality` function exactly what state the repository is in. It can only be in one of two states: `branch` or `patch`. `branch` means the patch has not been applied. The `patch` state is after the patch has been applied.
+The `repostatus` parameter tells the `personality` function exactly what state the source repository is in. It can only be in one of two states: `branch` or `patch`. `branch` means the patch has not been applied. The `patch` state is after the patch has been applied.
The `testtype` state tells the personality exactly which test is about to be executed.
In order to communicate back to test-patch, there are two functions for the personality to use.
-The first is `clear_personality_queue`. This removes the previous test's configuration so that a new module queue may be built.
+The first is `clear_personality_queue`. This removes the previous test's configuration so that a new module queue may be built. Custom personality_modules will almost always want to do this as the first action.
The second is `personality_enqueue_module`. This function takes two parameters. The first parameter is the name of the module to add to this test's queue. The second parameter is an option list of additional flags to pass to Maven when processing it. `personality_enqueue_module` may be called as many times as necessary for your project.
NOTE: A module name of . signifies the root of the repository.
-For example, let's say your project uses a special configuration to skip unit tests (-DskipTests). Running unit tests during a javadoc build isn't very interesting. We can write a simple personality check to disable the unit tests:
+For example, let's say your project uses a special configuration to skip unit tests (-DskipTests). Running unit tests during a javadoc build isn't very useful and wastes a lot of time. We can write a simple personality check to disable the unit tests:
```bash
@@ -160,5 +173,32 @@ function personality_modules
```
-This function will tell test-patch that when the javadoc test is being run, do the documentation test at the base of the repository and make sure the -DskipTests flag is passed to Maven.
+This function will tell test-patch that when the javadoc test is being run, do the documentation build at the base of the source repository and make sure the -DskipTests flag is passed to our build tool.
+
+
+
+# Important Variables
+
+There are a handful of extremely important variables that make life easier for personality and plug-in writers:
+
+* BUILD\_NATIVE will be set to true if the system has requested that non-JVM-based code be built (e.g., JNI or other compiled C code). Under Jenkins, this is always true.
+
+* BUILDTOOL specifies which tool is currently being used to drive compilation. Additionally, many build tools define xyz\_ARGS to pass on to the build tool command line. (e.g., MAVEN\_ARGS if maven is in use). Projects may set this in their personality. NOTE: today, only one build tool at a time is supported. This may change in the future.
+
+* CHANGED\_FILES is a list of all files that appear to be added, deleted, or modified in the patch.
+
+* CHANGED\_UNFILTERED\_MODULES is a list of all modules that house all of the CHANGED\_FILES. Be aware that the root of the source tree is reported as '.'.
+
+* CHANGED\_MODULES reports which modules that appear to have source code in them.
+
+* HOW\_TO\_CONTRIBUTE should be a URL that points to a project's on-boarding documentation for new users. Currently, it is used to suggest a review of patch naming guidelines. Since this should be project specific information, it is useful to set in a project's personality.
+
+* ISSUE\_RE is to help test-patch when talking to JIRA. It helps determine if the given project is appropriate for the given JIRA issue.
+
+* MODULE and other MODULE\_\* are arrays that contain which modules, the status, etc, to be operated upon. These should be treated as read-only by plug-ins.
+
+* PATCH\_BRANCH\_DEFAULT is the name of the branch in the git repo that is considered the master. This is useful to set in personalities.
+
+* PATCH\_DIR is the name of the temporary directory that houses test-patch artifacts (such as logs and the patch file itself)
+* TEST\_PARALLEL if parallel unit tests have been requested. Project personalities are responsible for actually enabling or ignoring the request. TEST\_THREADS is the number of threads that have been requested to run in parallel.
http://git-wip-us.apache.org/repos/asf/yetus/blob/84fea273/dev-support/docs/precommit-architecture.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-architecture.md b/dev-support/docs/precommit-architecture.md
index c134728..cd527ae 100644
--- a/dev-support/docs/precommit-architecture.md
+++ b/dev-support/docs/precommit-architecture.md
@@ -14,17 +14,17 @@
# Some Philosophy
-* Everyone's time is valuable. The quicker contributors can get feedback and iterate, the more likely their contribution will get checked in. A committer should be able to focus on the core issues of a contribution rather than details that might be able to be determined automatically.
+* Everyone's time is valuable. The quicker contributors can get feedback and iterate, the more likely and faster their contribution will get checked in. A committer should be able to focus on the core issues of a contribution rather than details that can be determined automatically.
-* Precommit checks should be fast. There is no value in testing parts of the source tree that are not immediately impacted by a change. Unit testing is the target. They are not a replacement for full builds, which is where integration tests should happen.
+* Precommit checks should be fast. There is no value in testing parts of the source tree that are not immediately impacted by a change. Unit testing is the target. They are not a replacement for full builds or integration tests.
* Many open source projects have a desire to have this capability. Why not generalize a solution?
* In many build systems (especially with maven), a modular design has been picked. Why not leverage that design to make checks faster?
-* Projects that use the same language will, with a high degree of certainity, benefit from the same types of checks.
+* Projects that use the same language will, with a high degree of certainty, benefit from the same types of checks.
-* Portability matters.
+* Portability matters. Tooling should be as operating system and language agnostic as possible.
# Phases
@@ -32,7 +32,7 @@ test-patch works effectively under several different phases:
## Setup
-This is where test-patch configures and validates the environemnt. Some things done in this phase:
+This is where test-patch configures and validates the environment. Some things done in this phase:
* Defaults
* Parameter handling
@@ -50,12 +50,12 @@ This acts as a verification of all of the setup parts and is the final place to
## Pre-apply
-This is where the 'before' work is handled. Some things done in this phase:
+This is where the 'before' work is handled. Some things that typically get checked in this phase:
* The first pass of files and modules that will get patched
-* Validation and information gathering of java, javadoc, site, the mvn repo, findbugs, etc.
+* Validation and information gathering of the source tree pre-patch
* Author checks
-* check for modified unit tests
+* Check for modified unit tests
## Patch is Applied
@@ -65,15 +65,14 @@ The patch gets applied. Then a second pass to determine which modules and files
Now that the patch has been applied, many of the same checks performed in the Pre-apply step are done again to build an 'after' picture.
-* Validation and information gathering of java, javadoc, site, the mvn repo, findbugs, etc.
-
## Post-install
Some tests only work correctly when the repo is up-to-date. So
-mvn install is run to update the local repo and we enter this phase. Tests performed here:
+mvn install is run to update the local repo and we enter this phase. Some example tests performed here:
-* Verification that maven eclipse integration still works
-* FindBugs
+* javadoc
+* Findbugs
+* Maven eclipse integration still works
## Unit Tests
@@ -81,7 +80,7 @@ Since unit tests are generally the slowest part of the precommit process, they a
## Reporting
-Finally, the results are reported to the screen and, optionally, to JIRA.
+Finally, the results are reported to the screen and, optionally, to JIRA and/or whatever bug system has been configured.
# Test Flow
@@ -90,8 +89,8 @@ The basic workflow for many of the sub-items in individual phases are:
1. print a header, so the end user knows that something is happening
1. verify if the test is needed. If so, continue on. Otherwise, return success and let the next part of the phase execute.
1. Ask the personality about what modules and what flags should get used
-1. Execute maven in the given modules with the given flags. Log the output and record the time and result code.
-1. Do any extra work as appropriate (diffs, counts, etc) and either accept the status and message given by the maven run or change the vote, message, log file, etc.
+1. Execute maven (or some other build tool) in the given modules with the given flags. Log the output and record the time and result code.
+1. Do any extra work as appropriate (diffs, counts, etc) and either accept the status and message given by the maven run or change the vote, message, log file, etc, based upon this extra work.
1. Add the outcome(s) to the report generator
As one can see, the modules list is one of the key inputs into what actually gets executed. As a result, projects must full flexibility in either adding, modifying, or even removing modules from the test list. If a personality removes the entire list of modules, then that test should just be ignored.
http://git-wip-us.apache.org/repos/asf/yetus/blob/84fea273/dev-support/docs/precommit-basic.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-basic.md b/dev-support/docs/precommit-basic.md
index a830cdb..e68ad07 100644
--- a/dev-support/docs/precommit-basic.md
+++ b/dev-support/docs/precommit-basic.md
@@ -18,25 +18,30 @@ test-patch
* [Purpose](#Purpose)
* [Pre-requisites](#Pre-requisites)
* [Basic Usage](#Basic_Usage)
+* [Build Tool](#Build_Tool)
+* [Providing Patch Files](#Providing_Patch_Files)
+* [Project-Specific Capabilities](#Project-Specific_Capabilities)
+* [MultiJDK](#MultiJDK)
+* [Docker](#Docker)
-## Purpose
+# Purpose
-As part of Hadoop's commit process, all patches to the source base go through a precommit test that does some (usually) light checking to make sure the proposed change does not break unit tests and/or passes some other prerequisites. This is meant as a preliminary check for committers so that the basic patch is in a known state. This check, called test-patch, may also be used by individual developers to verify a patch prior to sending to the Hadoop QA systems.
+As part of Hadoop's commit process, all patches to the source base go through a precommit test that does some (relatively) light checking to make sure the proposed change does not break unit tests and/or passes some other prerequisites such as code formatting guidelines. This is meant as a preliminary check for committers so that the basic patch is in a known state and for contributors to know if they have followed the project's guidelines. This check, called test-patch, may also be used by individual developers to verify a patch prior to sending to the Hadoop QA systems.
Other projects have adopted a similar methodology after seeing great success in the Hadoop model. Some have even gone as far as forking Hadoop's precommit code and modifying it to meet their project's needs.
This is a modification to Hadoop's version of test-patch so that we may bring together all of these forks under a common code base to help the community as a whole.
-## Pre-requisites
+# Pre-requisites
test-patch has the following requirements:
* Ant- or Maven-based project (and ant/maven installed)
-* git-based project (and git installed)
+* git-based project (and git 1.7.3 or higher installed)
* bash v3.2 or higher
* findbugs 3.x installed
-* shellcheck installed
+* shellcheck installed, preferably 0.3.6 or higher
* pylint installed
* GNU diff
* GNU patch
@@ -57,21 +62,21 @@ Optional:
* Apache JIRA-based issue tracking
* JIRA cli tools
-The locations of these files are (mostly) assumed to be in the file path, but may be overridden via command line options. For Solaris and Solaris-like operating systems, the default location for the POSIX binaries is in /usr/xpg4/bin.
+The locations of these files are (mostly) assumed to be in the file path, but may be overridden via command line options. For Solaris and Solaris-like operating systems, the default location for the POSIX binaries is in /usr/xpg4/bin and the default location for the GNU binaries is /usr/gnu/bin.
-## Basic Usage
+# Basic Usage
-This command will execute basic patch testing against a patch file stored in filename:
+This command will execute basic patch testing against a patch file stored in "filename":
```bash
$ cd <your repo>
$ dev-support/test-patch.sh --dirty-workspace --project=projectname <filename>
```
-The `--dirty-workspace` flag tells test-patch that the repository is not clean and it is ok to continue. This version command does not run the unit tests.
+The `--dirty-workspace` flag tells test-patch that the repository is not clean and it is ok to continue. By default, unit tests are not run since they may take a significant amount of time.
-To do that, we need to provide the --run-tests command:
+To do turn them on, we need to provide the --run-tests option:
```bash
@@ -85,16 +90,34 @@ A typical configuration is to have two repositories. One with the code you are
```bash
$ cd <workrepo>
-$ git diff --no-prefix trunk > /tmp/patchfile
+$ git diff master > /tmp/patchfile
$ cd ../<testrepo>
$ <workrepo>/dev-support/test-patch.sh --basedir=<testrepo> --resetrepo /tmp/patchfile
```
We used two new options here. --basedir sets the location of the repository to use for testing. --resetrepo tells test patch that it can go into **destructive** mode. Destructive mode will wipe out any changes made to that repository, so use it with care!
-After the tests have run, there is a directory that contains all of the test-patch related artifacts. This is generally referred to as the patchprocess directory. By default, test-patch tries to make something off of /tmp to contain this content. Using the `--patchdir` command, one can specify exactly which directory to use. This is helpful for automated precommit testing so that the Jenkins or other automated workflow system knows where to look to gather up the output.
+After the tests have run, there is a directory that contains all of the test-patch related artifacts. This is generally referred to as the patchprocess directory. By default, test-patch tries to make something off of /tmp to contain this content. Using the `--patch-dir` option, one can specify exactly which directory to use. This is helpful for automated precommit testing so that Jenkins or other automated workflow system knows where to look to gather up the output.
-## Providing Patch Files
+For example:
+
+```bash
+$ test-patch.sh --jenkins --patch-dir=${WORKSPACE}/patchprocess --basedir=${WORKSPACE}/source ${WORKSPACE}/patchfile
+```
+
+... will trigger test-patch to run in fully automated Jenkins mode, using ${WORKSPACE}/patchprocess as its scratch space, ${WORKSPACE}/source as the source repository, and ${WORKSPACE}/patchfile as the name of the patch to test against.
+
+# Build Tool
+
+Out of the box, test-patch is built to use maven. But what if the project is built using something else, such as ant?
+
+```bash
+$ test-patch.sh (other options) --build-tool=ant
+```
+
+will tell test-patch to use ant instead of maven to drive the project.
+
+# Providing Patch Files
It is a fairly common practice within the Apache community to use Apache's JIRA instance to store potential patches. As a result, test-patch supports providing just a JIRA issue number. test-patch will find the *last* attachment, download it, then process it.
@@ -106,7 +129,6 @@ $ test-patch.sh (other options) HADOOP-9905
... will process the patch file associated with this JIRA issue.
-
A new practice is to use a service such as GitHub and its Pull Request (PR) feature. Luckily, test-patch supports URLs and many services like GitHub provide ways to provide unified diffs via URLs.
For example:
@@ -117,6 +139,50 @@ $ test-patch.sh (other options) https://github.com/apache/flink/pull/773.patch
... will grab a unified diff of PR #773 and process it.
+# Project-specific Capabilities
+
+Due to the extensible nature of the system, test-patch allows for projects to define project-specific rules which we call personalities. (How to build those rules is covered elsewhere.) There are two ways to specify which personality to use:
+
+## Direct Method
+
+```bash
+$ test-patch.sh (other options) --personality=(filename)
+```
+
+This tells test-patch to use the personality in the given file.
+
+## Project Method
+
+However, test-patch can detect if it is a personality that is in its "personality" directory based upon the project name:
+
+```bash
+$ test-patch.sh (other options) --project=(project)
+```
+
+# MultiJDK
+
+For many projects, it is useful to test Java code against multiple versions of JDKs at the same time. test-patch can do this with the --multijdkdirs option:
+
+```bash
+$ test-patch.sh (other options) --multijdkdirs="/j/d/k/1,/j/d/k/2"
+```
+
+Not all Java tests support this mode, but those that do will now run their tests with all of the given versions of Java consecutively (e.g., javac--the Java compliation test). Tests that do not support MultiJDK mode (e.g., checkstyle, mvn install) will use JAVA\_HOME.
+
+NOTE: JAVA\_HOME is always appended to the list of JDKs in MultiJDK mode. If JAVA\_HOME is in the list, it will be moved to the end.
+
+# Docker
+
+test-patch also has a mode to utilize Docker:
+
+```bash
+$ test-patch.sh (other options) --docker
+```
+
+This will do some preliminary setup and then re-execute itself inside a Docker container. For more information on how to provide a custom Dockerfile, see the advanced guide.
+
+
+
## In Closing
test-patch has many other features and command line options for the basic user. Many of these are self-explanatory. To see the list of options, run test-patch.sh without any options or with --help.
[26/50] [abbrv] yetus git commit: HADOOP-12315. hbaseprotoc_postapply
in the test-patch hbase personality can return a wrong status (Kengo Seki via
aw)
Posted by bu...@apache.org.
HADOOP-12315. hbaseprotoc_postapply in the test-patch hbase personality can return a wrong status (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/d32053a6
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/d32053a6
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/d32053a6
Branch: refs/heads/master
Commit: d32053a67272cc1c1873ba614ca9df1828ab32c4
Parents: 495f95e
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 13 12:51:00 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 13 12:51:00 2015 -0700
----------------------------------------------------------------------
dev-support/personality/hbase.sh | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/d32053a6/dev-support/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hbase.sh b/dev-support/personality/hbase.sh
index d8ca901..9749096 100755
--- a/dev-support/personality/hbase.sh
+++ b/dev-support/personality/hbase.sh
@@ -146,7 +146,7 @@ function hbaseprotoc_postapply
local module
local logfile
local count
- local results
+ local result
big_console_header "Patch HBase protoc plugin"
@@ -159,7 +159,7 @@ function hbaseprotoc_postapply
fi
personality_modules patch hbaseprotoc
- modules_workers patch hbaseprotoc -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
+ modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
# shellcheck disable=SC2153
until [[ $i -eq ${#MODULE[@]} ]]; do
@@ -177,13 +177,13 @@ function hbaseprotoc_postapply
if [[ ${count} -gt 0 ]]; then
module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
"${count} new protoc errors in ${module}."
- ((results=results+1))
+ ((result=result+1))
fi
((i=i+1))
done
modules_messages patch hbaseprotoc true
- if [[ ${results} -gt 0 ]]; then
+ if [[ ${result} -gt 0 ]]; then
return 1
fi
return 0
[43/50] [abbrv] yetus git commit: HADOOP-12380. Wrong grep command
invocation in github_find_jira_title (Brahma Reddy Battula via aw)
Posted by bu...@apache.org.
HADOOP-12380. Wrong grep command invocation in github_find_jira_title (Brahma Reddy Battula via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/6f0772d8
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/6f0772d8
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/6f0772d8
Branch: refs/heads/master
Commit: 6f0772d8872e74b3f4189963272bc673108c1640
Parents: 677bfe0
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Sep 8 11:27:18 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Sep 8 11:27:18 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/github.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/6f0772d8/dev-support/test-patch.d/github.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/github.sh b/dev-support/test-patch.d/github.sh
index f742062..e4aa0e5 100755
--- a/dev-support/test-patch.d/github.sh
+++ b/dev-support/test-patch.d/github.sh
@@ -136,7 +136,7 @@ function github_find_jira_title
return 1
fi
- title=$(GREP title "${PATCH_DIR}/github-pull.json" \
+ title=$(${GREP} title "${PATCH_DIR}/github-pull.json" \
| cut -f4 -d\")
# people typically do two types: JIRA-ISSUE: and [JIRA-ISSUE]
[25/50] [abbrv] yetus git commit: HADOOP-12244. recover broken rebase
during precommit (aw)
Posted by bu...@apache.org.
HADOOP-12244. recover broken rebase during precommit (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/495f95e7
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/495f95e7
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/495f95e7
Branch: refs/heads/master
Commit: 495f95e788fdc58121781d45b96a69b69c34cf2a
Parents: 91cbda7
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 13 12:30:41 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 13 12:30:41 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/495f95e7/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 4dd15e6..13332b7 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1290,6 +1290,12 @@ function git_checkout
# we need to explicitly fetch in case the
# git ref hasn't been brought in tree yet
if [[ ${OFFLINE} == false ]]; then
+
+ if [[ -f .git/rebase-apply ]]; then
+ yetus_error "ERROR: previous rebase failed. Aborting it."
+ ${GIT} rebase --abort
+ fi
+
${GIT} pull --rebase
if [[ $? != 0 ]]; then
yetus_error "ERROR: git pull is failing"
[45/50] [abbrv] yetus git commit: HADOOP-12277. releasedocmaker index
mode should create a readme.md in addition to a index.md (aw)
Posted by bu...@apache.org.
HADOOP-12277. releasedocmaker index mode should create a readme.md in addition to a index.md (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/e63656ba
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/e63656ba
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/e63656ba
Branch: refs/heads/master
Commit: e63656bad3cb437e636e5171d6f67b0ad58d8dd0
Parents: e54a49b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Sep 9 08:40:18 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Sep 9 08:40:18 2015 -0700
----------------------------------------------------------------------
dev-support/docs/releasedocmaker.md | 9 +++++++++
dev-support/releasedocmaker.py | 26 +++++++++++++++++++++-----
2 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/e63656ba/dev-support/docs/releasedocmaker.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/releasedocmaker.md b/dev-support/docs/releasedocmaker.md
index 31eb8e0..405800d 100644
--- a/dev-support/docs/releasedocmaker.md
+++ b/dev-support/docs/releasedocmaker.md
@@ -21,6 +21,7 @@ releasedocmaker
* [Multiple Versions](#Multiple_Versions)
* [Unreleased Dates](#Unreleased_Dates)
* [Lint Mode](#Lint_Mode)
+* [Index Mode](#Index_Mode)
# Purpose
@@ -119,3 +120,11 @@ $ releasedocmaker.py --project HBASE --version 1.0.0 --lint
```
This will do the normal JIRA querying, looking for items it considers problematic. It will print the information to the screen and then exit with either success or failure, depending upon if any issues were discovered.
+
+# Index Mode
+
+There is basic support for an autoindexer. It will create two files that contain links to all directories that have a major.minor.micro-style version numbering system, where all fields are numeric.
+
+ * index.md: a file suitable for conversion to HTML via mvn site
+ * README.md: a file suitable for display on Github and other Markdown rendering websites
+
http://git-wip-us.apache.org/repos/asf/yetus/blob/e63656ba/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index d62c0d1..0c16d8a 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -19,6 +19,7 @@
from glob import glob
from optparse import OptionParser
from time import gmtime, strftime
+from distutils.version import LooseVersion
import os
import re
import sys
@@ -113,24 +114,38 @@ def mstr(obj):
return unicode(obj)
def buildindex(title, asf_license):
- versions = reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
+ """Write an index file for later conversion using mvn site"""
+ versions = glob("[0-9]*.[0-9]*.[0-9]*")
+ versions.sort(key=LooseVersion, reverse=True)
with open("index.md", "w") as indexfile:
if asf_license is True:
indexfile.write(ASF_LICENSE)
for version in versions:
indexfile.write("* %s v%s\n" % (title, version))
for k in ("Changes", "Release Notes"):
- indexfile.write(" * %s (%s/%s.%s.html)\n" \
+ indexfile.write(" * [%s](%s/%s.%s.html)\n" \
+ % (k, version, k.upper().replace(" ", ""), version))
+
+def buildreadme(title, asf_license):
+ """Write an index file for Github using README.md"""
+ versions = glob("[0-9]*.[0-9]*.[0-9]*")
+ versions.sort(key=LooseVersion, reverse=True)
+ with open("README.md", "w") as indexfile:
+ if asf_license is True:
+ indexfile.write(ASF_LICENSE)
+ for version in versions:
+ indexfile.write("* %s v%s\n" % (title, version))
+ for k in ("Changes", "Release Notes"):
+ indexfile.write(" * [%s](%s/%s.%s.md)\n" \
% (k, version, k.upper().replace(" ", ""), version))
- indexfile.close()
class GetVersions(object):
- """ yo """
+ """ List of version strings """
def __init__(self, versions, projects):
versions = versions
projects = projects
self.newversions = []
- versions.sort()
+ versions.sort(key=LooseVersion)
print "Looking for %s through %s"%(versions[0], versions[-1])
for project in projects:
url = "https://issues.apache.org/jira/rest/api/2/project/%s/versions" % project
@@ -587,6 +602,7 @@ def main():
if options.index:
buildindex(title, options.license)
+ buildreadme(title, options.license)
if haderrors is True:
sys.exit(1)
[22/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/e207ab52
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/e207ab52
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/e207ab52
Branch: refs/heads/master
Commit: e207ab5289397123a921bb9f628b8b3a44b55cd6
Parents: 74e2b2f c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 11 07:44:21 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Aug 11 07:44:21 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[14/50] [abbrv] yetus git commit: HADOOP-12121. smarter branch
detection (aw)
Posted by bu...@apache.org.
HADOOP-12121. smarter branch detection (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/634bf711
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/634bf711
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/634bf711
Branch: refs/heads/master
Commit: 634bf7119481e1fe586a0976fe5595285f9d65d6
Parents: fe51d66
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 3 10:47:11 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 3 10:47:11 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 160 ++++++++++++++++++++++++-----------------
1 file changed, 94 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/634bf711/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 48f83a0..1f1f88e 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1281,9 +1281,6 @@ function git_checkout
fi
determine_branch
- if [[ ${PATCH_BRANCH} =~ ^git ]]; then
- PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
- fi
# we need to explicitly fetch in case the
# git ref hasn't been brought in tree yet
@@ -1322,9 +1319,6 @@ function git_checkout
fi
determine_branch
- if [[ ${PATCH_BRANCH} =~ ^git ]]; then
- PATCH_BRANCH=$(echo "${PATCH_BRANCH}" | cut -dt -f2)
- fi
currentbranch=$(${GIT} rev-parse --abbrev-ref HEAD)
if [[ "${currentbranch}" != "${PATCH_BRANCH}" ]];then
@@ -1351,20 +1345,19 @@ function git_checkout
return 0
}
-## @description Confirm the given branch is a member of the list of space
-## @description delimited branches or a git ref
+## @description Confirm the given branch is a git reference
+## @descriptoin or a valid gitXYZ commit hash
## @audience private
## @stability evolving
## @replaceable no
## @param branch
-## @param branchlist
-## @return 0 on success
+## @return 0 on success, if gitXYZ was passed, PATCH_BRANCH=xyz
## @return 1 on failure
function verify_valid_branch
{
- local branches=$1
- local check=$2
+ local check=$1
local i
+ local hash
# shortcut some common
# non-resolvable names
@@ -1372,26 +1365,22 @@ function verify_valid_branch
return 1
fi
- if [[ ${check} == patch ]]; then
- return 1
- fi
-
if [[ ${check} =~ ^git ]]; then
- ref=$(echo "${check}" | cut -f2 -dt)
- count=$(echo "${ref}" | wc -c | tr -d ' ')
-
- if [[ ${count} == 8 || ${count} == 41 ]]; then
- return 0
+ hash=$(echo "${check}" | cut -f2- -dt)
+ if [[ -n ${hash} ]]; then
+ ${GIT} cat-file -t "${hash}" >/dev/null 2>&1
+ if [[ $? -eq 0 ]]; then
+ PATCH_BRANCH=${hash}
+ return 0
+ fi
+ return 1
+ else
+ return 1
fi
- return 1
fi
- for i in ${branches}; do
- if [[ "${i}" == "${check}" ]]; then
- return 0
- fi
- done
- return 1
+ ${GIT} show-ref "${check}" >/dev/null 2>&1
+ return $?
}
## @description Try to guess the branch being tested using a variety of heuristics
@@ -1402,10 +1391,9 @@ function verify_valid_branch
## @return 1 on failure, with PATCH_BRANCH updated to PATCH_BRANCH_DEFAULT
function determine_branch
{
- local allbranches
local patchnamechunk
-
- yetus_debug "Determine branch"
+ local total
+ local count
# something has already set this, so move on
if [[ -n ${PATCH_BRANCH} ]]; then
@@ -1414,6 +1402,13 @@ function determine_branch
pushd "${BASEDIR}" > /dev/null
+ yetus_debug "Determine branch"
+
+ # something has already set this, so move on
+ if [[ -n ${PATCH_BRANCH} ]]; then
+ return
+ fi
+
# developer mode, existing checkout, whatever
if [[ "${DIRTY_WORKSPACE}" == true ]];then
PATCH_BRANCH=$(${GIT} rev-parse --abbrev-ref HEAD)
@@ -1421,50 +1416,83 @@ function determine_branch
return
fi
- allbranches=$(${GIT} branch -r | tr -d ' ' | ${SED} -e s,origin/,,g)
-
for j in "${PATCHURL}" "${PATCH_OR_ISSUE}"; do
- yetus_debug "Determine branch: starting with ${j}"
- # shellcheck disable=SC2016
- patchnamechunk=$(echo "${j}" | ${AWK} -F/ '{print $NF}')
-
- # ISSUE.branch.##.patch
- yetus_debug "Determine branch: ISSUE.branch.##.patch"
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2 -d. )
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
+ if [[ -z "${j}" ]]; then
+ continue
fi
-
- # ISSUE-branch-##.patch
- yetus_debug "Determine branch: ISSUE-branch-##.patch"
+ yetus_debug "Determine branch: starting with ${j}"
+ patchnamechunk=$(echo "${j}" \
+ | ${SED} -e 's,.*/\(.*\)$,\1,' \
+ -e 's,\.txt,.,' \
+ -e 's,.patch,.,g' \
+ -e 's,.diff,.,g' \
+ -e 's,\.\.,.,g' \
+ -e 's,\.$,,g' )
+
+ # ISSUE-branch-##
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1,2 -d-)
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
+ yetus_debug "Determine branch: ISSUE-branch-## = ${PATCH_BRANCH}"
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return
+ fi
fi
- # ISSUE-##.patch.branch
- yetus_debug "Determine branch: ISSUE-##.patch.branch"
- # shellcheck disable=SC2016
- PATCH_BRANCH=$(echo "${patchnamechunk}" | ${AWK} -F. '{print $NF}')
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
+ # ISSUE-##[.##].branch
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d. )
+ count="${PATCH_BRANCH//[^.]}"
+ total=${#count}
+ ((total = total + 3 ))
+ until [[ ${total} -eq 2 ]]; do
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3-${total} -d.)
+ yetus_debug "Determine branch: ISSUE[.##].branch = ${PATCH_BRANCH}"
+ ((total=total-1))
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return
+ fi
+ fi
+ done
+
+ # ISSUE.branch.##
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2- -d. )
+ count="${PATCH_BRANCH//[^.]}"
+ total=${#count}
+ ((total = total + 3 ))
+ until [[ ${total} -eq 2 ]]; do
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f2-${total} -d.)
+ yetus_debug "Determine branch: ISSUE.branch[.##] = ${PATCH_BRANCH}"
+ ((total=total-1))
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return
+ fi
+ fi
+ done
+
+ # ISSUE-branch.##
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1- -d. )
+ count="${PATCH_BRANCH//[^.]}"
+ total=${#count}
+ ((total = total + 1 ))
+ until [[ ${total} -eq 1 ]]; do
+ PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1-${total} -d. )
+ yetus_debug "Determine branch: ISSUE-branch[.##] = ${PATCH_BRANCH}"
+ ((total=total-1))
+ if [[ -n "${PATCH_BRANCH}" ]]; then
+ verify_valid_branch "${PATCH_BRANCH}"
+ if [[ $? == 0 ]]; then
+ return
+ fi
+ fi
+ done
- # ISSUE-branch.##.patch
- yetus_debug "Determine branch: ISSUE-branch.##.patch"
- # shellcheck disable=SC2016
- PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | ${AWK} -F. '{print $(NF-2)}' 2>/dev/null)
- verify_valid_branch "${allbranches}" "${PATCH_BRANCH}"
- if [[ $? == 0 ]]; then
- return
- fi
done
PATCH_BRANCH="${PATCH_BRANCH_DEFAULT}"
-
popd >/dev/null
}
[21/50] [abbrv] yetus git commit: HADOOP-12248. Add native support
for TAP (aw)
Posted by bu...@apache.org.
HADOOP-12248. Add native support for TAP (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/74e2b2f5
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/74e2b2f5
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/74e2b2f5
Branch: refs/heads/master
Commit: 74e2b2f570829f9d78ccb8d1705e4c386512488c
Parents: b571374
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 10 11:10:36 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 10 11:10:36 2015 -0700
----------------------------------------------------------------------
dev-support/personality/hadoop.sh | 74 ++++++++++++++++++++
dev-support/test-patch.d/builtin-personality.sh | 22 +++---
dev-support/test-patch.d/shellcheck.sh | 2 +-
dev-support/test-patch.sh | 40 +++++++++--
4 files changed, 122 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/74e2b2f5/dev-support/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hadoop.sh b/dev-support/personality/hadoop.sh
index 60dbb3d..1243a17 100755
--- a/dev-support/personality/hadoop.sh
+++ b/dev-support/personality/hadoop.sh
@@ -253,6 +253,26 @@ function personality_modules
#fi
needflags=true
hadoop_unittest_prereqs
+
+ verify_needed_test javac
+ if [[ $? == 0 ]]; then
+ yetus_debug "hadoop: javac not requested"
+ verify_needed_test native
+ if [[ $? == 0 ]]; then
+ yetus_debug "hadoop: native not requested"
+ yetus_debug "hadoop: adding -DskipTests to unit test"
+ extra="-DskipTests"
+ fi
+ fi
+
+ verify_needed_test shellcheck
+ if [[ $? == 0
+ && ! ${CHANGED_FILES} =~ \.bats ]]; then
+ yetus_debug "hadoop: NO shell code change detected; disabling shelltest profile"
+ extra="${extra} -P!shelltest"
+ else
+ extra="${extra} -Pshelltest"
+ fi
;;
*)
extra="-DskipTests"
@@ -272,3 +292,57 @@ function personality_modules
done
}
+function personality_file_tests
+{
+ local filename=$1
+
+ yetus_debug "Using Hadoop-specific personality_file_tests"
+
+ if [[ ${filename} =~ src/main/webapp ]]; then
+ yetus_debug "tests/webapp: ${filename}"
+ elif [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ || ${filename} =~ src/scripts
+ || ${filename} =~ src/test/scripts
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ add_test unit
+ elif [[ ${filename} =~ \.md$
+ || ${filename} =~ \.md\.vm$
+ || ${filename} =~ src/site
+ ]]; then
+ yetus_debug "tests/site: ${filename}"
+ add_test site
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test cc
+ add_test unit
+ add_test javac
+ elif [[ ${filename} =~ build.xml$
+ || ${filename} =~ pom.xml$
+ || ${filename} =~ \.java$
+ || ${filename} =~ src/main
+ ]]; then
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test javac
+ add_test javadoc
+ add_test mvninstall
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ src/test ]]; then
+ yetus_debug "tests"
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/74e2b2f5/dev-support/test-patch.d/builtin-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/builtin-personality.sh b/dev-support/test-patch.d/builtin-personality.sh
index dc944e4..4be3bfa 100755
--- a/dev-support/test-patch.d/builtin-personality.sh
+++ b/dev-support/test-patch.d/builtin-personality.sh
@@ -55,6 +55,8 @@ function builtin_mvn_personality_file_tests
yetus_debug "tests/webapp: ${filename}"
elif [[ ${filename} =~ \.sh
|| ${filename} =~ \.cmd
+ || ${filename} =~ src/main/scripts
+ || ${filename} =~ src/test/scripts
]]; then
yetus_debug "tests/shell: ${filename}"
elif [[ ${filename} =~ \.md$
@@ -69,29 +71,31 @@ function builtin_mvn_personality_file_tests
|| ${filename} =~ \.h$
|| ${filename} =~ \.hh$
|| ${filename} =~ \.proto$
- || ${filename} =~ src/test
|| ${filename} =~ \.cmake$
|| ${filename} =~ CMakeLists.txt
]]; then
yetus_debug "tests/units: ${filename}"
+ add_test cc
+ add_test unit
+ elif [[ ${filename} =~ \.scala$ ]]; then
add_test javac
- add_test mvninstall
add_test unit
- elif [[ ${filename} =~ pom.xml$
+ add_test mvninstall
+ elif [[ ${filename} =~ build.xml$
+ || ${filename} =~ pom.xml$
|| ${filename} =~ \.java$
- || ${filename} =~ \.scala$
|| ${filename} =~ src/main
]]; then
- if [[ ${filename} =~ src/main/bin
- || ${filename} =~ src/main/sbin ]]; then
- yetus_debug "tests/shell: ${filename}"
- else
yetus_debug "tests/javadoc+units: ${filename}"
add_test javac
add_test javadoc
add_test mvninstall
add_test unit
- fi
+ fi
+
+ if [[ ${filename} =~ src/test ]]; then
+ yetus_debug "tests"
+ add_test unit
fi
if [[ ${filename} =~ \.java$ ]]; then
http://git-wip-us.apache.org/repos/asf/yetus/blob/74e2b2f5/dev-support/test-patch.d/shellcheck.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/shellcheck.sh b/dev-support/test-patch.d/shellcheck.sh
index 14d1d18..4d17768 100755
--- a/dev-support/test-patch.d/shellcheck.sh
+++ b/dev-support/test-patch.d/shellcheck.sh
@@ -56,7 +56,7 @@ function shellcheck_private_findbash
fi
list="${list} ${i}"
done
- done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name libexec -o -type d -name shellprofile.d)
+ done < <(find . -type d -name bin -o -type d -name sbin -o -type d -name scripts -o -type d -name libexec -o -type d -name shellprofile.d)
# shellcheck disable=SC2086
echo ${list} ${SHELLCHECK_SPECIFICFILES} | tr ' ' '\n' | sort -u
}
http://git-wip-us.apache.org/repos/asf/yetus/blob/74e2b2f5/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 446d5cf..4dd15e6 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -30,6 +30,8 @@ BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
STARTINGDIR=$(pwd)
USER_PARAMS=("$@")
GLOBALTIMER=$(date +"%s")
+#shellcheck disable=SC2034
+QATESTMODE=false
# global arrays
declare -a MAVEN_ARGS=("--batch-mode")
@@ -1177,6 +1179,7 @@ function find_changed_modules
#shellcheck disable=SC2086,SC2116
CHANGED_UNFILTERED_MODULES=$(echo ${CHANGED_UNFILTERED_MODULES})
+
if [[ ${BUILDTOOL} = maven ]]; then
# Filter out modules without code
for module in ${builddirs}; do
@@ -1786,7 +1789,7 @@ function copytpbits
# if we've already copied, then don't bother doing it again
if [[ ${STARTDIR} == ${PATCH_DIR}/precommit ]]; then
- hadoop_debug "Skipping copytpbits; already copied once"
+ yetus_debug "Skipping copytpbits; already copied once"
return
fi
@@ -2942,6 +2945,7 @@ function populate_test_table
function check_unittests
{
local i
+ local testsys
local test_logfile
local result=0
local -r savejavahome=${JAVA_HOME}
@@ -2949,6 +2953,9 @@ function check_unittests
local jdk=""
local jdkindex=0
local statusjdk
+ local formatresult=0
+ local needlog
+ local unitlogs
big_console_header "Running unit tests"
@@ -2976,7 +2983,7 @@ function check_unittests
personality_modules patch unit
case ${BUILDTOOL} in
maven)
- modules_workers patch unit clean install -fae
+ modules_workers patch unit clean test -fae
;;
ant)
modules_workers patch unit
@@ -3002,13 +3009,23 @@ function check_unittests
pushd "${MODULE[${i}]}" >/dev/null
- for j in ${TESTSYSTEMS}; do
- if declare -f ${j}_process_tests; then
- "${j}_process_tests" "${module}" "${test_logfile}"
- ((results=results+$?))
+ needlog=0
+ for testsys in ${TESTFORMATS}; do
+ if declare -f ${testsys}_process_tests >/dev/null; then
+ yetus_debug "Calling ${testsys}_process_tests"
+ "${testsys}_process_tests" "${module}" "${test_logfile}" "${fn}"
+ formatresult=$?
+ ((results=results+formatresult))
+ if [[ "${formatresult}" != 0 ]]; then
+ needlog=1
+ fi
fi
done
+ if [[ ${needlog} == 1 ]]; then
+ unitlogs="${unitlogs} @@BASE@@/patch-unit-${fn}.txt"
+ fi
+
popd >/dev/null
((i=i+1))
@@ -3017,10 +3034,21 @@ function check_unittests
done
JAVA_HOME=${savejavahome}
+ if [[ -n "${unitlogs}" ]]; then
+ add_footer_table "unit test logs" "${unitlogs}"
+ fi
+
if [[ ${JENKINS} == true ]]; then
add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"
fi
+ for testsys in ${TESTFORMATS}; do
+ if declare -f ${testsys}_finalize_results >/dev/null; then
+ yetus_debug "Calling ${testsys}_finalize_results"
+ "${testsys}_finalize_results" "${statusjdk}"
+ fi
+ done
+
if [[ ${result} -gt 0 ]]; then
return 1
fi
[32/50] [abbrv] yetus git commit: HADOOP-12339. wrong help message
about --curl-cmd option. (Gabor Liptak via busbey)
Posted by bu...@apache.org.
HADOOP-12339. wrong help message about --curl-cmd option. (Gabor Liptak via busbey)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/42fd07cb
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/42fd07cb
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/42fd07cb
Branch: refs/heads/master
Commit: 42fd07cb0cb60eafefafa787dbc0091a97dde71b
Parents: 463891f
Author: Sean Busbey <bu...@apache.org>
Authored: Wed Aug 19 21:51:09 2015 -0500
Committer: Sean Busbey <bu...@cloudera.com>
Committed: Wed Aug 19 21:51:25 2015 -0500
----------------------------------------------------------------------
dev-support/test-patch.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/42fd07cb/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 67dc3b4..4d4b63f 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -811,7 +811,7 @@ function testpatch_usage
echo "Shell binary overrides:"
echo "--ant-cmd=<cmd> The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')"
echo "--awk-cmd=<cmd> The 'awk' command to use (default 'awk')"
- echo "--curl-cmd=<cmd> The 'wget' command to use (default 'curl')"
+ echo "--curl-cmd=<cmd> The 'curl' command to use (default 'curl')"
echo "--diff-cmd=<cmd> The GNU-compatible 'diff' command to use (default 'diff')"
echo "--file-cmd=<cmd> The 'file' command to use (default 'file')"
echo "--git-cmd=<cmd> The 'git' command to use (default 'git')"
[24/50] [abbrv] yetus git commit: HADOOP-12244. recover broken rebase
during precommit (aw)
Posted by bu...@apache.org.
HADOOP-12244. recover broken rebase during precommit (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/8369b1cc
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/8369b1cc
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/8369b1cc
Branch: refs/heads/master
Commit: 8369b1cc0192ecf29c8ba4a81a10ae7f29f98ef1
Parents: c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 13 12:29:19 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 13 12:29:19 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/8369b1cc/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index efcd614..a3cdc85 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -947,6 +947,12 @@ function git_checkout
# we need to explicitly fetch in case the
# git ref hasn't been brought in tree yet
if [[ ${OFFLINE} == false ]]; then
+
+ if [[ -f .git/rebase-apply ]]; then
+ hadoop_error "ERROR: previous rebase failed. Aborting it."
+ ${GIT} rebase --abort
+ fi
+
${GIT} pull --rebase
if [[ $? != 0 ]]; then
hadoop_error "ERROR: git pull is failing"
[15/50] [abbrv] yetus git commit: add missing files from previous
commits because I can never remember to do a git add
Posted by bu...@apache.org.
add missing files from previous commits because I can never remember to do a git add
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/a627ff03
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/a627ff03
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/a627ff03
Branch: refs/heads/master
Commit: a627ff039bedb9f0a1ff27bc970a272568a14fe2
Parents: 634bf71
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 3 11:03:18 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 3 11:03:18 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/perlcritic.sh | 140 ++++++++++++++++++++++++++++
dev-support/test-patch.d/ruby-lint.sh | 140 ++++++++++++++++++++++++++++
2 files changed, 280 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/a627ff03/dev-support/test-patch.d/perlcritic.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/perlcritic.sh b/dev-support/test-patch.d/perlcritic.sh
new file mode 100755
index 0000000..1cec3f3
--- /dev/null
+++ b/dev-support/test-patch.d/perlcritic.sh
@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin perlcritic
+
+PERLCRITIC_TIMER=0
+
+PERLCRITIC=${PERLCRITIC:-$(which perlcritic 2>/dev/null)}
+
+function perlcritic_usage
+{
+ echo "Perl::Critic specific:"
+ echo "--perlcritic=<path> path to perlcritic executable"
+}
+
+function perlcritic_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --perlcritic=*)
+ PERLCRITIC=${i#*=}
+ ;;
+ esac
+ done
+}
+
+function perlcritic_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.p[lm]$ ]]; then
+ add_test perlcritic
+ fi
+}
+
+function perlcritic_preapply
+{
+ local i
+
+ verify_needed_test perlcritic
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "Perl::Critic plugin: prepatch"
+
+ if [[ ! -x ${PERLCRITIC} ]]; then
+ yetus_error "${PERLCRITIC} does not exist."
+ return 0
+ fi
+
+ start_clock
+
+ echo "Running perlcritic against modified perl scripts/modules."
+ pushd "${BASEDIR}" >/dev/null
+ for i in ${CHANGED_FILES}; do
+ if [[ ${i} =~ \.p[lm]$ && -f ${i} ]]; then
+ ${PERLCRITIC} -1 --verbose 1 "${i}" 2>/dev/null >> "${PATCH_DIR}/branch-perlcritic-result.txt"
+ fi
+ done
+ popd >/dev/null
+ # keep track of how much as elapsed for us already
+ PERLCRITIC_TIMER=$(stop_clock)
+ return 0
+}
+
+function perlcritic_postapply
+{
+ local i
+ local numPrepatch
+ local numPostpatch
+ local diffPostpatch
+
+ verify_needed_test perlcritic
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "Perl::Critic plugin: postpatch"
+
+ if [[ ! -x ${PERLCRITIC} ]]; then
+ yetus_error "${PERLCRITIC} is not available."
+ add_vote_table 0 perlcritic "Perl::Critic was not available."
+ return 0
+ fi
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${PERLCRITIC_TIMER}"
+
+ echo "Running perlcritic against modified perl scripts/modules."
+ # we re-check this in case one has been added
+ pushd "${BASEDIR}" >/dev/null
+ for i in ${CHANGED_FILES}; do
+ if [[ ${i} =~ \.p[lm]$ && -f ${i} ]]; then
+ ${PERLCRITIC} -1 --verbose 1 "${i}" 2>/dev/null >> "${PATCH_DIR}/patch-perlcritic-result.txt"
+ fi
+ done
+ popd >/dev/null
+
+ PERLCRITIC_VERSION=$(${PERLCRITIC} --version 2>/dev/null)
+ add_footer_table perlcritic "v${PERLCRITIC_VERSION}"
+
+ calcdiffs "${PATCH_DIR}/branch-perlcritic-result.txt" "${PATCH_DIR}/patch-perlcritic-result.txt" > "${PATCH_DIR}/diff-patch-perlcritic.txt"
+ # shellcheck disable=SC2016
+ diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-perlcritic.txt" | ${AWK} '{print $1}')
+
+ if [[ ${diffPostpatch} -gt 0 ]] ; then
+ # shellcheck disable=SC2016
+ numPrepatch=$(wc -l "${PATCH_DIR}/branch-perlcritic-result.txt" | ${AWK} '{print $1}')
+
+ # shellcheck disable=SC2016
+ numPostpatch=$(wc -l "${PATCH_DIR}/patch-perlcritic-result.txt" | ${AWK} '{print $1}')
+
+ add_vote_table -1 perlcritic "The applied patch generated "\
+ "${diffPostpatch} new Perl::Critic issues (total was ${numPrepatch}, now ${numPostpatch})."
+ add_footer_table perlcritic "@@BASE@@/diff-patch-perlcritic.txt"
+ return 1
+ fi
+
+ add_vote_table +1 perlcritic "There were no new perlcritic issues."
+ return 0
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/a627ff03/dev-support/test-patch.d/ruby-lint.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/ruby-lint.sh b/dev-support/test-patch.d/ruby-lint.sh
new file mode 100755
index 0000000..35d9604
--- /dev/null
+++ b/dev-support/test-patch.d/ruby-lint.sh
@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin ruby_lint
+
+RUBY_LINT_TIMER=0
+
+RUBY_LINT=${RUBY_LINT:-$(which ruby-lint 2>/dev/null)}
+
+function ruby_lint_usage
+{
+ echo "Ruby-lint specific:"
+ echo "--ruby-lint=<path> path to ruby-lint executable"
+}
+
+function ruby_lint_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --ruby-lint=*)
+ RUBY_LINT=${i#*=}
+ ;;
+ esac
+ done
+}
+
+function ruby_lint_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.rb$ ]]; then
+ add_test ruby_lint
+ fi
+}
+
+function ruby_lint_preapply
+{
+ local i
+
+ verify_needed_test ruby_lint
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "ruby-lint plugin: prepatch"
+
+ if [[ ! -x ${RUBY_LINT} ]]; then
+ yetus_error "${RUBY_LINT} does not exist."
+ return 0
+ fi
+
+ start_clock
+
+ echo "Running ruby-lint against modified ruby scripts."
+ pushd "${BASEDIR}" >/dev/null
+ for i in ${CHANGED_FILES}; do
+ if [[ ${i} =~ \.rb$ && -f ${i} ]]; then
+ ${RUBY_LINT} -p syntastic "${i}" | sort -t : -k 1,1 -k 3,3n -k 4,4n >> "${PATCH_DIR}/branch-ruby-lint-result.txt"
+ fi
+ done
+ popd >/dev/null
+ # keep track of how much as elapsed for us already
+ RUBY_LINT_TIMER=$(stop_clock)
+ return 0
+}
+
+function ruby_lint_postapply
+{
+ local i
+ local numPrepatch
+ local numPostpatch
+ local diffPostpatch
+
+ verify_needed_test ruby_lint
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ big_console_header "ruby-lint plugin: postpatch"
+
+ if [[ ! -x ${RUBY_LINT} ]]; then
+ yetus_error "${RUBY_LINT} is not available."
+ add_vote_table 0 ruby-lint "Ruby-lint was not available."
+ return 0
+ fi
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${RUBY_LINT_TIMER}"
+
+ echo "Running ruby-lint against modified ruby scripts."
+ # we re-check this in case one has been added
+ pushd "${BASEDIR}" >/dev/null
+ for i in ${CHANGED_FILES}; do
+ if [[ ${i} =~ \.rb$ && -f ${i} ]]; then
+ ${RUBY_LINT} -p syntastic "${i}" | sort -t : -k 1,1 -k 3,3n -k 4,4n >> "${PATCH_DIR}/patch-ruby-lint-result.txt"
+ fi
+ done
+ popd >/dev/null
+
+ # shellcheck disable=SC2016
+ RUBY_LINT_VERSION=$(${RUBY_LINT} -v | ${AWK} '{print $2}')
+ add_footer_table ruby-lint "${RUBY_LINT_VERSION}"
+
+ calcdiffs "${PATCH_DIR}/branch-ruby-lint-result.txt" "${PATCH_DIR}/patch-ruby-lint-result.txt" > "${PATCH_DIR}/diff-patch-ruby-lint.txt"
+ diffPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-ruby-lint.txt")
+
+ if [[ ${diffPostpatch} -gt 0 ]] ; then
+ # shellcheck disable=SC2016
+ numPrepatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-ruby-lint-result.txt")
+
+ # shellcheck disable=SC2016
+ numPostpatch=$(${AWK} -F: 'BEGIN {sum=0} 4<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-ruby-lint-result.txt")
+
+ add_vote_table -1 ruby-lint "The applied patch generated "\
+ "${diffPostpatch} new ruby-lint issues (total was ${numPrepatch}, now ${numPostpatch})."
+ add_footer_table ruby-lint "@@BASE@@/diff-patch-ruby-lint.txt"
+ return 1
+ fi
+
+ add_vote_table +1 ruby-lint "There were no new ruby-lint issues."
+ return 0
+}
[36/50] [abbrv] yetus git commit: HADOOP-12301. Fix some test-patch
plugins to count the diff lines correctly (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12301. Fix some test-patch plugins to count the diff lines correctly (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/12532050
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/12532050
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/12532050
Branch: refs/heads/master
Commit: 12532050e56177a70a94a6d76645928cbd4826ce
Parents: 90b8e66
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 25 09:16:59 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Aug 25 09:16:59 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/pylint.sh | 6 +++---
dev-support/test-patch.d/rubocop.sh | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/12532050/dev-support/test-patch.d/pylint.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/pylint.sh b/dev-support/test-patch.d/pylint.sh
index 1ee5a8f..6fa576e 100755
--- a/dev-support/test-patch.d/pylint.sh
+++ b/dev-support/test-patch.d/pylint.sh
@@ -148,14 +148,14 @@ function pylint_postapply
add_footer_table pylint "v${PYLINT_VERSION%,}"
calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" "${PATCH_DIR}/patch-pylint-result.txt" > "${PATCH_DIR}/diff-patch-pylint.txt"
- diffPostpatch=$(${AWK} 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/diff-patch-pylint.txt")
+ diffPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/diff-patch-pylint.txt")
if [[ ${diffPostpatch} -gt 0 ]] ; then
# shellcheck disable=SC2016
- numPrepatch=$(${AWK} 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/branch-pylint-result.txt")
+ numPrepatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/branch-pylint-result.txt")
# shellcheck disable=SC2016
- numPostpatch=$(${AWK} 'BEGIN {sum=0} 2<NF {sum+=1} END {print sum}' "${PATCH_DIR}/patch-pylint-result.txt")
+ numPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/patch-pylint-result.txt")
add_vote_table -1 pylint "The applied patch generated "\
"${diffPostpatch} new pylint issues (total was ${numPrepatch}, now ${numPostpatch})."
http://git-wip-us.apache.org/repos/asf/yetus/blob/12532050/dev-support/test-patch.d/rubocop.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/rubocop.sh b/dev-support/test-patch.d/rubocop.sh
index ba9810e..091a2b1 100755
--- a/dev-support/test-patch.d/rubocop.sh
+++ b/dev-support/test-patch.d/rubocop.sh
@@ -70,7 +70,7 @@ function rubocop_preapply
pushd "${BASEDIR}" >/dev/null
for i in ${CHANGED_FILES}; do
if [[ ${i} =~ \.rb$ && -f ${i} ]]; then
- ${RUBOCOP} -f c "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/branch-rubocop-result.txt"
+ ${RUBOCOP} -f e "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/branch-rubocop-result.txt"
fi
done
popd >/dev/null
@@ -110,7 +110,7 @@ function rubocop_postapply
pushd "${BASEDIR}" >/dev/null
for i in ${CHANGED_FILES}; do
if [[ ${i} =~ \.rb$ && -f ${i} ]]; then
- ${RUBOCOP} -f c "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/patch-rubocop-result.txt"
+ ${RUBOCOP} -f e "${i}" | ${AWK} '!/[0-9]* files? inspected/' >> "${PATCH_DIR}/patch-rubocop-result.txt"
fi
done
popd >/dev/null
[28/50] [abbrv] yetus git commit: HADOOP-12297. test-patch's basedir
and patch-dir must be directories under the user's home in docker mode if
using boot2docker (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12297. test-patch's basedir and patch-dir must be directories under the user's home in docker mode if using boot2docker (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/5821d5c6
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/5821d5c6
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/5821d5c6
Branch: refs/heads/master
Commit: 5821d5c6afe5efbed9926adf8b9dff556ed0a84e
Parents: 942def2
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Aug 13 12:56:33 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 13 12:56:33 2015 -0700
----------------------------------------------------------------------
dev-support/docs/precommit-advanced.md | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/5821d5c6/dev-support/docs/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-advanced.md b/dev-support/docs/precommit-advanced.md
index a424199..3185512 100644
--- a/dev-support/docs/precommit-advanced.md
+++ b/dev-support/docs/precommit-advanced.md
@@ -28,6 +28,8 @@ By default, test-patch runs in the same shell where it was launched. It can alt
The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. However be aware that test-patch will copy this file and append its necessary hooks to re-launch itself prior to executing docker.
+NOTE: If you are using Boot2Docker, you must use directories under /Users (OSX) or C:\Users (Windows) as the base and patchprocess directories (specified by the --basedir and --patch-dir options respectively), because automatically mountable directories are limited to them. See [the Docker documentation](https://docs.docker.com/userguide/dockervolumes/#mount-a-host-directory-as-a-data-volume).
+
Dockerfile images will be named with a test-patch prefix and suffix with either a date or a git commit hash. By using this information, test-patch will automatically manage broken/stale container images that are hanging around if it is run in --jenkins mode. In this way, if Docker fails to build the image, the disk space should eventually be cleaned and returned back to the system.
# Maven Specific
[50/50] [abbrv] yetus git commit: HADOOP-12400. Wrong comment for
scaladoc_rebuild function in test-patch scala plugin (Jagadesh Kiran N via
aw)
Posted by bu...@apache.org.
HADOOP-12400. Wrong comment for scaladoc_rebuild function in test-patch scala plugin (Jagadesh Kiran N via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/2db6e7a1
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/2db6e7a1
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/2db6e7a1
Branch: refs/heads/master
Commit: 2db6e7a1af3152f24c2aeb97d38623ff2745b23d
Parents: 1c4921c
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Sep 11 18:02:59 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Sep 11 18:02:59 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/scala.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/2db6e7a1/dev-support/test-patch.d/scala.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/scala.sh b/dev-support/test-patch.d/scala.sh
index 4c88db4..09e8dde 100755
--- a/dev-support/test-patch.d/scala.sh
+++ b/dev-support/test-patch.d/scala.sh
@@ -59,7 +59,7 @@ function scalac_compile
fi
}
-## @description Count and compare the number of JavaDoc warnings pre- and post- patch
+## @description Count and compare the number of ScalaDoc warnings pre- and post- patch
## @audience private
## @stability evolving
## @replaceable no
[47/50] [abbrv] yetus git commit: HADOOP-12257. rework build tool
support; add gradle; add scala (aw)
Posted by bu...@apache.org.
HADOOP-12257. rework build tool support; add gradle; add scala (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/02226269
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/02226269
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/02226269
Branch: refs/heads/master
Commit: 02226269bef256e1d1baa5665c79312183b7125b
Parents: e63656b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Sep 9 10:21:32 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Sep 9 10:21:32 2015 -0700
----------------------------------------------------------------------
dev-support/docs/precommit-advanced.md | 99 +-
dev-support/docs/precommit-architecture.md | 42 +-
dev-support/docs/precommit-basic.md | 5 +-
dev-support/docs/precommit-bugsystems.md | 46 +
dev-support/docs/precommit-buildtools.md | 100 +
dev-support/personality/bigtop.sh | 67 +
dev-support/personality/flink.sh | 61 +-
dev-support/personality/hadoop.sh | 27 +-
dev-support/personality/hbase.sh | 113 +-
dev-support/personality/kafka.sh | 58 +
dev-support/personality/pig.sh | 2 +-
dev-support/personality/samza.sh | 26 +
dev-support/personality/tajo.sh | 40 +-
dev-support/personality/tez.sh | 44 -
dev-support/test-patch.d/ant.sh | 183 ++
dev-support/test-patch.d/asflicense.sh | 25 +-
dev-support/test-patch.d/author.sh | 52 +
dev-support/test-patch.d/builtin-bugsystem.sh | 0
dev-support/test-patch.d/builtin-personality.sh | 140 +-
dev-support/test-patch.d/checkstyle.sh | 59 +-
dev-support/test-patch.d/findbugs.sh | 53 +-
dev-support/test-patch.d/gradle.sh | 248 +++
dev-support/test-patch.d/java.sh | 158 ++
dev-support/test-patch.d/jira.sh | 2 +-
dev-support/test-patch.d/maven.sh | 328 +++
dev-support/test-patch.d/perlcritic.sh | 11 +
dev-support/test-patch.d/pylint.sh | 11 +
dev-support/test-patch.d/rubocop.sh | 11 +
dev-support/test-patch.d/ruby-lint.sh | 11 +
dev-support/test-patch.d/scala.sh | 77 +
dev-support/test-patch.d/shellcheck.sh | 11 +
dev-support/test-patch.d/test4tests.sh | 58 +
dev-support/test-patch.d/whitespace.sh | 20 +-
dev-support/test-patch.d/xml.sh | 15 +-
dev-support/test-patch.sh | 1934 +++++++-----------
35 files changed, 2455 insertions(+), 1682 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/docs/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-advanced.md b/dev-support/docs/precommit-advanced.md
index 7830afe..83bda30 100644
--- a/dev-support/docs/precommit-advanced.md
+++ b/dev-support/docs/precommit-advanced.md
@@ -16,8 +16,6 @@ test-patch
==========
* [Docker Support](#Docker_Support)
-* [Maven Specific](#Maven_Specific)
-* [Ant Specific](#Ant_Specific)
* [Plug-ins](#Plug-ins)
* [Configuring for Other Projects](#Configuring_for_Other_Projects)
* [Important Variables](#Important_Variables)
@@ -32,107 +30,71 @@ NOTE: If you are using Boot2Docker, you must use directories under /Users (OSX)
Dockerfile images will be named with a test-patch prefix and suffix with either a date or a git commit hash. By using this information, test-patch will automatically manage broken/stale container images that are hanging around if it is run in --jenkins mode. In this way, if Docker fails to build the image, the disk space should eventually be cleaned and returned back to the system.
-# Maven Specific
-
-## Command Arguments
-
-test-patch always passes --batch-mode to maven to force it into non-interactive mode. Additionally, some tests will also force -fae in order to get all of messages/errors during that mode. It *does not* pass -DskipTests. Additional arguments should be handled via the personality.
-
-## Test Profile
-
-By default, test-patch will pass -Ptest-patch to Maven. This will allow you to configure special actions that should only happen when running underneath test-patch.
-
-# Ant Specific
-
-## Command Arguments
-
-test-patch always passes -noinput to Ant. This force ant to be non-interactive.
# Plug-ins
-test-patch allows one to add to its basic feature set via plug-ins. There is a directory called test-patch.d off of the directory where test-patch.sh lives. Inside this directory one may place some bash shell fragments that, if setup with proper functions, will allow for test-patch to call it as necessary.
+test-patch allows one to add to its basic feature set via plug-ins. There is a directory called test-patch.d off of the directory where test-patch.sh lives. Inside this directory one may place some bash shell fragments that, if setup with proper functions, will allow for test-patch to call it as necessary. Different plug-ins have specific functions for that particular functionality. In this document, the common functions as well as test functions are covered. See other documentat for pertinent information for the other plug-in types.
-## Test Plug-ins
+## Common Plug-in Functions
-Every test plugin must have one line in order to be recognized:
+Every plug-in must have one line in order to be recognized, usually an 'add' statement. Test plug-ins, for example, have this statement:
```bash
add_plugin <pluginname>
```
-This function call registers the `pluginname` so that test-patch knows that it exists. This plug-in name also acts as the key to the custom functions that you can define. For example:
+This function call registers the `pluginname` so that test-patch knows that it exists. The `pluginname` also acts as the key to the custom functions that you can define. For example:
```bash
function pluginname_filefilter
```
-This function gets called for every file that a patch may contain. This allows the plug-in author to determine if this plug-in should be called, what files it might need to analyze, etc.
+defines the filefilter for the `pluginname` plug-in.
Similarly, there are other functions that may be defined during the test-patch run:
-* pluginname\_postcheckout
- - executed prior to the patch being applied but after the git repository is setup. This is useful for any early error checking that might need to be done before any heavier work.
-
-* pluginname\_preapply
- - executed prior to the patch being applied. This is useful for any "before"-type data collection for later comparisons.
-
-* pluginname\_postapply
- - executed after the patch has been applied. This is useful for any "after"-type data collection.
-
-* pluginname\_postinstall
- - executed after the mvn install test has been done. If any tests require the Maven repository to be up-to-date with the contents of the patch, this is the place.
-
-* pluginname\_tests
- - executed after the unit tests have completed.
-
-If the plug-in has some specific options, one can use following functions:
-
-* pluginname\_usage
-
- - executed when the help message is displayed. This is used to display the plug-in specific options for the user.
-
-* pluginname\_parse\_args
-
- - executed prior to any other above functions except for pluginname_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
-
HINT: It is recommended to make the pluginname relatively small, 10 characters at the most. Otherwise, the ASCII output table may be skewed.
-## Bug System Plug-ins
-
-Similar to tests, the ability to add support for bug tracking systems is also handled via a plug-in mechanism.
-
-* pluginname_usage
+* pluginname\_initialize
+ - After argument parsing and prior to any other work, the initialize step allows a plug-in to do any precursor work, set internal defaults, etc.
+* pluginname\_usage
- executed when the help message is displayed. This is used to display the plug-in specific options for the user.
* pluginname\_parse\_args
+ - executed prior to any other above functions except for pluginname\_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
- - executed prior to any other above functions except for pluginname_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
-
-
-* pluginname\_locate\_patch
+* pluginname\_precheck
+ - executed prior to the patch being applied but after the git repository is setup. Returning a fail status here will exit test-patch.
- - Given input from the user, download the patch if possible.
+* pluginname\_precompile
+ - executed prior to the compilation part of the lifecycle. This is useful for doing setup work required by the compilation process.
-* pluginname\_determine\_branch
+* pluginname\_compile
+ - executed immediately after the actual compilation. This is step is intended to be used to verify the results and add extra checking of the compile phase and it's stdout/stderr.
- - Using any heuristics available, return the branch to process, if possible.
+* pluginname\_postcompile
+ - This step happens after the compile phase.
-* pluginname\_determine\_issue
+* pluginname\_rebuild
+ - Any non-unit tests that require the source to be rebuilt in a destructive way should be run here.
- - Using any heuristics available, set the issue, bug number, etc, for this bug system, if possible. This is typically used to fill in supplementary information in the final output table.
+Test Plug-ins
+=============
-* pluginname\_writecomment
+Plugins geared towards independent tests are registed via:
- - Given text input, write this output to the bug system as a comment. NOTE: It is the bug system's responsibility to format appropriately.
-* pluginname\_linecomments
+```bash
+add_plugin <pluginname>
+```
- - This function allows for the system to write specific comments on specific lines if the bug system supports code review comments.
++ pluginname\_filefilter
+ - executed while determine which files trigger which tests. This function should use 'add_test pluginname' to add the plug-in to the test list.
-* pluginname\_finalreport
+* pluginname\_tests
+ - executed after the unit tests have completed.
- - Write the final result table to the bug system.
# Configuring for Other Projects
@@ -197,7 +159,6 @@ The second is `personality_enqueue_module`. This function takes two parameters.
For example, let's say your project uses a special configuration to skip unit tests (-DskipTests). Running unit tests during a javadoc build isn't very useful and wastes a lot of time. We can write a simple personality check to disable the unit tests:
-
```bash
function personality_modules
{
@@ -214,8 +175,6 @@ function personality_modules
This function will tell test-patch that when the javadoc test is being run, do the documentation build at the base of the source repository and make sure the -DskipTests flag is passed to our build tool.
-
-
# Important Variables
There are a handful of extremely important system variables that make life easier for personality and plug-in writers. Other variables may be provided by individual plug-ins. Check their development documentation for more information.
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/docs/precommit-architecture.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-architecture.md b/dev-support/docs/precommit-architecture.md
index cd527ae..e0a178a 100644
--- a/dev-support/docs/precommit-architecture.md
+++ b/dev-support/docs/precommit-architecture.md
@@ -30,49 +30,55 @@
test-patch works effectively under several different phases:
-## Setup
+## Initialize
This is where test-patch configures and validates the environment. Some things done in this phase:
* Defaults
* Parameter handling
-* Importing plugins and personalities
+* Importing plug-ins and personalities
* Docker container launching
* Re-exec support
* Patch file downloading
* git repository management (fresh pull, branch switching, etc)
-## Post-checkout
+## Precheck
Checks done here are *fatal*.
This acts as a verification of all of the setup parts and is the final place to short-cut the full test cycle. The most significant built-in check done here is verifying the patch file is a valid.
-## Pre-apply
+## Patch File Tests
-This is where the 'before' work is handled. Some things that typically get checked in this phase:
+Tests that only require the patch file are run. Note that the repository is still from the initial checkout!
+
+## Compile Cycle (Branch)
+
+When compilation must be done, we follow these five steps:
+
+* The list of modules that require analysis is built.
+* A precompile step to set things up for the actual compile
+* The actual compile
+* A postcompile to do analysis on the output of that compile phase
+* A rebuild phase to run tests that require recompiles
+
+The first time this is done is with the pristine checkout. This is called the "branch compile". For this pass, this is where the 'before' work is handled. Some things that typically get checked in this phase:
* The first pass of files and modules that will get patched
* Validation and information gathering of the source tree pre-patch
-* Author checks
-* Check for modified unit tests
-
-## Patch is Applied
+* javadoc, scaladoc, etc
-The patch gets applied. Then a second pass to determine which modules and files have been changed in order to handle any modules that might have added or moved.
+## Distribution Clean
-## Post-apply
+This step is to wipe the repository clean back to a pristine state such that the previous cycle will not impact the next cycle.
-Now that the patch has been applied, many of the same checks performed in the Pre-apply step are done again to build an 'after' picture.
+## Patch Application
-## Post-install
+The patch gets applied.
-Some tests only work correctly when the repo is up-to-date. So
-mvn install is run to update the local repo and we enter this phase. Some example tests performed here:
+## Compile Cycle (Patch)
-* javadoc
-* Findbugs
-* Maven eclipse integration still works
+Now that the patch has been applied the steps to compile we outlined in the compilation (branch) phase are repeated but with the patch applied. This is where a lot of 'after' checks are performed.
## Unit Tests
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/docs/precommit-basic.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-basic.md b/dev-support/docs/precommit-basic.md
index a612214..9cb4035 100644
--- a/dev-support/docs/precommit-basic.md
+++ b/dev-support/docs/precommit-basic.md
@@ -37,7 +37,7 @@ This is a modification to Hadoop's version of test-patch so that we may bring to
test-patch has the following requirements:
-* Ant- or Maven-based project (and ant/maven installed)
+* A project with a supported build tool (ant, gradle, maven, ...)
* git-based project (and git 1.7.3 or higher installed)
* bash v3.2 or higher
* findbugs 3.x installed
@@ -52,7 +52,7 @@ test-patch has the following requirements:
* file command
* smart-apply-patch.sh (included!)
-Maven plugins requirements:
+Maven plug-ins requirements:
* Apache RAT
* Apache FindBugs
@@ -61,6 +61,7 @@ Optional:
* JIRA-based issue tracking
* GitHub-based issue tracking
+* Some other supported bug system
The locations of these files are (mostly) assumed to be in the file path, but may be overridden via command line options. For Solaris and Solaris-like operating systems, the default location for the POSIX binaries is in /usr/xpg4/bin and the default location for the GNU binaries is /usr/gnu/bin.
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/docs/precommit-bugsystems.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-bugsystems.md b/dev-support/docs/precommit-bugsystems.md
new file mode 100644
index 0000000..6af1266
--- /dev/null
+++ b/dev-support/docs/precommit-bugsystems.md
@@ -0,0 +1,46 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Bug System Support
+==================
+
+test-patch has the ability to support multiple bug systems. Bug tools have some extra hooks to fetch patches, line-level reporting, and posting a final report. Every bug system plug-in must have one line in order to be recognized:
+
+```bash
+add_bugsystem <pluginname>
+```
+
+* pluginname\_locate\_patch
+
+ - Given input from the user, download the patch if possible.
+
+* pluginname\_determine\_branch
+
+ - Using any heuristics available, return the branch to process, if possible.
+
+* pluginname\_determine\_issue
+
+ - Using any heuristics available, set the issue, bug number, etc, for this bug system, if possible. This is typically used to fill in supplementary information in the final output table.
+
+* pluginname\_writecomment
+
+ - Given text input, write this output to the bug system as a comment. NOTE: It is the bug system's responsibility to format appropriately.
+
+* pluginname\_linecomments
+
+ - This function allows for the system to write specific comments on specific lines if the bug system supports code review comments.
+
+* pluginname\_finalreport
+
+ - Write the final result table to the bug system.
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/docs/precommit-buildtools.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-buildtools.md b/dev-support/docs/precommit-buildtools.md
new file mode 100644
index 0000000..d14f34b
--- /dev/null
+++ b/dev-support/docs/precommit-buildtools.md
@@ -0,0 +1,100 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Build Tool Support
+===================
+
+test-patch has the ability to support multiple build tools. Build tool plug-ins have some extra hooks to do source and object maintenance at key points. Every build tool plug-in must have one line in order to be recognized:
+
+```bash
+add_build_tool <pluginname>
+```
+
+# Global Variables
+
+* BUILDTOOLCWD
+
+ - If the build tool does not always run from the ${BASEDIR} and instead needs to change the current working directory to work on a specific module, then set this to true. The default is false.
+
+* UNSUPPORTED_TEST
+
+ - If pluginname\_modules\_worker is given a test type that is not supported by the build system, set UNSUPPORTED_TEST=true. If it is supported, set UNSUPPORTED_TEST=false.
+
+For example, the gradle build tool does not have a standard way to execute checkstyle. So when checkstyle is requested, gradle\_modules\_worker sets UNSUPPORTED_TEST to true and returns out of the routine.
+
+# Required Functions
+
+* pluginname\_buildfile
+
+ - This should be an echo of the file that controls the build system. This is used for module determination.
+
+* pluginname\_executor
+
+ - This should be an echo of how to run the build tool, any extra arguments, etc.
+
+* pluginname\_modules\_worker
+
+ - Input is the branch and the test being run. This should call modules_workers with the generic parts to run that test on the build system. For example, if it is convention to use 'test' to trigger 'unit' tests, then module_workers should be called with 'test' appended onto its normal parameters.
+
+* pluginname\_builtin_personality\_modules
+
+ - Default method to determine how to enqueue modules for processing. Note that personalities may override this function.
+
+* pluginname\_builtin_personality\_file\_tests
+
+ - Default method to determine which tests to trigger. Note that personalities may override this function.
+
+# Optional Functions
+
+* pluginname\_postapply\_install
+
+ - After the install step, this allows the build tool plug-in to do extra work.
+
+* pluginname\_parse\_args
+
+ - executed prior to any other above functions except for pluginname\_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
+
+* pluginname\_initialize
+
+ - After argument parsing and prior to any other work, the initialize step allows a plug-in to do any precursor work, set internal defaults, etc.
+
+* pluginname\_count\_(test)\_probs
+
+ - Certain language test plug-ins require assistance from the build tool to count problems in the compile log due to some tools having custom handling for those languages. The test plug-in name should be in the (test) part of the function name.
+
+# Ant Specific
+
+## Command Arguments
+
+test-patch always passes -noinput to Ant. This forces ant to be non-interactive.
+
+# Gradle Specific
+
+The gradle plug-in always rebuilds the gradlew file and uses gradlew as the method to execute commands.
+
+# Maven Specific
+
+## Command Arguments
+
+test-patch always passes --batch-mode to maven to force it into non-interactive mode. Additionally, some tests will also force -fae in order to get all of messages/errors during that mode. Some tests are executed with -DskipTests. Additional arguments should be handled via the personality.
+
+## Test Profile
+
+By default, test-patch will pass -Ptest-patch to Maven. This will allow you to configure special actions that should only happen when running underneath test-patch.
+
+## Custom Maven Tests
+
+* Maven will trigger a maven install as part of the precompile.
+* Maven will test eclipse integration as part of the postcompile.
+* If src/site is modified, maven site tests are executed.
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/bigtop.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/bigtop.sh b/dev-support/personality/bigtop.sh
new file mode 100755
index 0000000..e467bd1
--- /dev/null
+++ b/dev-support/personality/bigtop.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=master
+# shellcheck disable=SC2034
+JIRA_ISSUE_RE='^BIGTOP-[0-9]+$'
+# shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE=""
+# shellcheck disable=SC2034
+BUILDTOOL=gradle
+# shellcheck disable=SC2034
+GITHUB_REPO="apache/bigtop"
+# shellcheck disable=SC2034
+BIGTOP_PUPPETSETUP=false
+
+add_plugin bigtop
+
+function bigtop_usage
+{
+ echo "Bigtop specific:"
+ echo "--bigtop-puppetsetup=[false|true] execute the bigtop dev setup (needs sudo to root)"
+}
+
+function bigtop_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --bigtop-puppet=*)
+ BIGTOP_PUPPETSETUP=${i#*=}
+ ;;
+ esac
+ done
+}
+
+function bigtop_precheck_postinstall
+{
+ if [[ ${BIGTOP_PUPPETSETUP} = "true" ]]; then
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/bigtop-branch-toolchain.txt" "${GRADLEW}" toolchain
+ popd >/dev/null
+ fi
+}
+
+function bigtop_postapply_postinstall
+{
+ if [[ ${BIGTOP_PUPPETSETUP} = "true" ]]; then
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/bigtop-patch-toolchain.txt" "${GRADLEW}" toolchain
+ popd >/dev/null
+ fi
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/flink.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/flink.sh b/dev-support/personality/flink.sh
index 4b6c390..fbf0ba0 100755
--- a/dev-support/personality/flink.sh
+++ b/dev-support/personality/flink.sh
@@ -57,7 +57,7 @@ function flinklib_preapply
pushd "${BASEDIR}" >/dev/null
echo_and_redirect "${PATCH_DIR}/branch-flinklib-root.txt" \
- "${MVN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
+ "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
if [[ $? != 0 ]]; then
add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}."
fi
@@ -78,7 +78,7 @@ function flinklib_postapply
pushd "${BASEDIR}" >/dev/null
echo_and_redirect "${PATCH_DIR}/patch-flinklib-root.txt" \
- "${MVN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
+ "${MAVEN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
FLINK_POST_LIB_FILES=$(flinklib_count)
popd >/dev/null
@@ -97,57 +97,14 @@ function flinklib_postapply
return 0
}
-function personality_modules
+function flinklib_rebuild
{
- local repostatus=$1
- local testtype=$2
- local extra=""
-
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- case ${testtype} in
- mvninstall)
- if [[ ${repostatus} == branch ]]; then
- personality_enqueue_module . -DskipTests
- return
- fi
- return
- ;;
- asflicense)
- # this is very fast and provides the full path if we do it from
- # the root of the source
- personality_enqueue_module .
- return
- ;;
- unit)
- if [[ ${TEST_PARALLEL} == "true" ]] ; then
- extra="-Pparallel-tests"
- if [[ -n ${TEST_THREADS:-} ]]; then
- extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
- fi
- fi
- ;;
- *)
- extra="-DskipTests"
- ;;
- esac
-
- for module in ${CHANGED_MODULES}; do
- # shellcheck disable=SC2086
- personality_enqueue_module ${module} ${extra}
- done
-}
-
-function personality_file_tests
-{
- local filename=$1
+ declare repostatus=$1
- yetus_debug "Using personality_file_tests, but calling the built-in:"
- builtin_personality_file_tests "${1}"
-
- if [[ ${filename} =~ \.scala$ ]]; then
- add_test unit
+ if [[ "${repostatus}" = branch ]]; then
+ flinklib_preapply
+ else
+ flinklib_postinstall
fi
}
+
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hadoop.sh b/dev-support/personality/hadoop.sh
index b3eb04a..0dd89db 100755
--- a/dev-support/personality/hadoop.sh
+++ b/dev-support/personality/hadoop.sh
@@ -34,6 +34,8 @@ function hadoop_module_manipulation
local startingmodules=${1:-normal}
local module
local hdfs_modules
+ local mapred_modules
+ local yarn_modules
local ordered_modules
local tools_modules
local passed_modules
@@ -67,9 +69,12 @@ function hadoop_module_manipulation
yetus_debug "hmm pre-ordering: ${startingmodules}"
- # yarn will almost always be after common in the sort order
- # so really just need to make sure that common comes before
- # everything else and tools comes last
+ # Hadoop's compilation rules are complex
+ # Precedence: common > [hdfs|yarn] > mapred > tools
+ # - everything depends upon common
+ # - hdfs needs common's native bits for unit tests
+ # - mapred depends upon yarn since it is a yarn app
+ # - tools can require anything
for module in ${passed_modules}; do
yetus_debug "Personality ordering ${module}"
@@ -85,12 +90,16 @@ function hadoop_module_manipulation
ordered_modules="${ordered_modules} ${module}"
elif [[ ${module} = hadoop-tools* ]]; then
tools_modules="${tools_modules} ${module}"
+ elif [[ ${module} = hadoop-mapreduce-project* ]]; then
+ mapred_modules="${mapred_modules} ${module}"
+ elif [[ ${module} = hadoop-yarn-project* ]]; then
+ yarn_modules="${yarn_modules} ${module}"
else
ordered_modules="${ordered_modules} ${module}"
fi
done
- HADOOP_MODULES="${ordered_modules} ${hdfs_modules} ${tools_modules}"
+ HADOOP_MODULES="${ordered_modules} ${hdfs_modules} ${yarn_modules} ${mapred_modules} ${tools_modules}"
yetus_debug "hmm out: ${HADOOP_MODULES}"
}
@@ -121,7 +130,7 @@ function hadoop_unittest_prereqs
pushd "${BASEDIR}/${module}" >/dev/null
# shellcheck disable=SC2086
echo_and_redirect "${PATCH_DIR}/maven-unit-prereq-${fn}-install.txt" \
- "${MVN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
+ "${MAVEN}" "${MAVEN_ARGS[@]}" install -DskipTests ${flags}
popd >/dev/null
fi
}
@@ -212,7 +221,7 @@ function personality_modules
ordering="union"
extra="-DskipTests"
;;
- javac)
+ compile)
ordering="union"
extra="-DskipTests"
needflags=true
@@ -223,6 +232,10 @@ function personality_modules
ordering="."
fi
;;
+ distclean)
+ ordering="."
+ extra="-DskipTests"
+ ;;
javadoc)
if [[ ${repostatus} = patch ]]; then
echo "javadoc pre-reqs:"
@@ -232,7 +245,7 @@ function personality_modules
pushd "${BASEDIR}/${i}" >/dev/null
echo "cd ${i}"
echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \
- "${MVN}" "${MAVEN_ARGS[@]}" install
+ "${MAVEN}" "${MAVEN_ARGS[@]}" install
popd >/dev/null
done
fi
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hbase.sh b/dev-support/personality/hbase.sh
index 4f23679..120efa7 100755
--- a/dev-support/personality/hbase.sh
+++ b/dev-support/personality/hbase.sh
@@ -24,7 +24,7 @@ GITHUB_REPO="apache/hbase"
HOW_TO_CONTRIBUTE=""
# All supported Hadoop versions that we want to test the compilation with
-HADOOP2_VERSIONS="2.4.1 2.5.2 2.6.0"
+HBASE_HADOOP_VERSIONS="2.4.1 2.5.2 2.6.0"
# Override the maven options
MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
@@ -39,48 +39,30 @@ function personality_modules
clear_personality_queue
- case ${testtype} in
- javac)
- personality_enqueue_module . -DskipTests
- return
- ;;
- mvninstall)
- extra="-DskipTests -DHBasePatchProcess"
- if [[ ${repostatus} == branch ]]; then
- personality_enqueue_module . "${extra}"
- return
+ extra="-DHBasePatchProcess"
+
+ if [[ ${repostatus} == branch
+ && ${testtype} == mvninstall ]];then
+ personality_enqueue_module . ${extra}
+ return
+ fi
+
+ if [[ ${testtype} = findbugs ]]; then
+ for module in ${CHANGED_MODULES}; do
+ # skip findbugs on hbase-shell
+ if [[ ${module} == hbase-shell ]]; then
+ continue
+ else
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
fi
- return
- ;;
- asflicense)
- # this is very fast and provides the full path if we do it from
- # the root of the source
- personality_enqueue_module . -DHBasePatchProcess
- return
- ;;
- unit)
- if [[ ${TEST_PARALLEL} == "true" ]] ; then
- extra="-Pparallel-tests -DHBasePatchProcess"
- if [[ -n ${TEST_THREADS:-} ]]; then
- extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
- fi
- fi
- ;;
- *)
- extra="-DskipTests -DHBasePatchProcess"
- ;;
- esac
+ done
+ return
+ fi
for module in ${CHANGED_MODULES}; do
-
- # skip findbugs on hbase-shell
- if [[ ${module} == hbase-shell
- && ${testtype} == findbugs ]]; then
- continue
- else
- # shellcheck disable=SC2086
- personality_enqueue_module ${module} ${extra}
- fi
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
done
}
@@ -97,25 +79,30 @@ function hadoopcheck_filefilter
fi
}
-function hadoopcheck_postapply
+function hadoopcheck_rebuild
{
- local HADOOP2_VERSION
+ local repostatus=$1
+ local hadoopver
local logfile
local count
local result=0
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
+
big_console_header "Compiling against various Hadoop versions"
export MAVEN_OPTS="${MAVEN_OPTS}"
- for HADOOP2_VERSION in ${HADOOP2_VERSIONS}; do
- logfile="${PATCH_DIR}/patch-javac-${HADOOP2_VERSION}.txt"
+ for hadoopver in ${HBASE_HADOOP_VERSIONS}; do
+ logfile="${PATCH_DIR}/patch-javac-${hadoopver}.txt"
echo_and_redirect "${logfile}" \
- "${MVN}" clean install \
+ "${MAVEN}" clean install \
-DskipTests -DHBasePatchProcess \
- -Dhadoop-two.version="${HADOOP2_VERSION}"
+ -Dhadoop-two.version="${hadoopver}"
count=$(${GREP} -c ERROR "${logfile}")
if [[ ${count} -gt 0 ]]; then
- add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${HADOOP2_VERSION}."
+ add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${hadoopver}."
((result=result+1))
fi
done
@@ -124,7 +111,7 @@ function hadoopcheck_postapply
return 1
fi
- add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HADOOP2_VERSIONS}."
+ add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HBASE_HADOOP_VERSIONS}."
return 0
}
@@ -141,7 +128,7 @@ function hbaseprotoc_filefilter
fi
}
-function hbaseprotoc_postapply
+function hbaseprotoc_rebuild
{
local i=0
local fn
@@ -150,16 +137,20 @@ function hbaseprotoc_postapply
local count
local result
- big_console_header "Patch HBase protoc plugin"
-
- start_clock
+ if [[ "${repostatus}" = branch ]]; then
+ return 0
+ fi
verify_needed_test hbaseprotoc
if [[ $? == 0 ]]; then
- echo "Patch does not need hbaseprotoc testing."
return 0
fi
+ big_console_header "Patch HBase protoc plugin"
+
+ start_clock
+
+
personality_modules patch hbaseprotoc
modules_workers patch hbaseprotoc compile -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
@@ -204,28 +195,28 @@ function hbaseanti_filefilter
fi
}
-function hbaseanti_preapply
+function hbaseanti_patchfile
{
+ local patchfile=$1
local warnings
local result
- big_console_header "Checking for known anti-patterns"
-
- start_clock
-
verify_needed_test hbaseanti
if [[ $? == 0 ]]; then
- echo "Patch does not need hbaseanti testing."
return 0
fi
- warnings=$(${GREP} 'new TreeMap<byte.*()' "${PATCH_DIR}/patch")
+ big_console_header "Checking for known anti-patterns"
+
+ start_clock
+
+ warnings=$(${GREP} 'new TreeMap<byte.*()' "${patchfile}")
if [[ ${warnings} -gt 0 ]]; then
add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}."
((result=result+1))
fi
- warnings=$(${GREP} 'import org.apache.hadoop.classification' "${PATCH_DIR}/patch")
+ warnings=$(${GREP} 'import org.apache.hadoop.classification' "${patchfile}")
if [[ ${warnings} -gt 0 ]]; then
add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}."
((result=result+1))
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/kafka.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/kafka.sh b/dev-support/personality/kafka.sh
new file mode 100755
index 0000000..83e5313
--- /dev/null
+++ b/dev-support/personality/kafka.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=trunk
+#shellcheck disable=SC2034
+JIRA_ISSUE_RE='^KAFKA-[0-9]+$'
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE="http://kafka.apache.org/contributing.html"
+# shellcheck disable=SC2034
+BUILDTOOL=gradle
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/kafka"
+
+function personality_modules
+{
+ declare repostatus=$1
+ declare testtype=$2
+ declare module
+ declare extra=""
+
+ yetus_debug "Using kafka personality_modules"
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ gradleboot)
+ # kafka's bootstrap is broken
+ if [[ ${testtype} == gradleboot ]]; then
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/kafka-configure-gradle.txt" gradle
+ popd >/dev/null
+ fi
+ ;;
+ compile)
+ extra="clean jar"
+ ;;
+ esac
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/pig.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/pig.sh b/dev-support/personality/pig.sh
index d67b227..91c29f8 100755
--- a/dev-support/personality/pig.sh
+++ b/dev-support/personality/pig.sh
@@ -43,7 +43,7 @@ function personality_modules
ANT_FINDBUGSXML="${BASEDIR}/build/test/findbugs/pig-findbugs-report.xml"
extra="-Dfindbugs.home=${FINDBUGS_HOME}"
;;
- javac)
+ compile)
extra="${extra} -Djavac.args=-Xlint -Dcompile.c++=yes clean piggybank"
;;
javadoc)
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/samza.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/samza.sh b/dev-support/personality/samza.sh
new file mode 100755
index 0000000..9d39d92
--- /dev/null
+++ b/dev-support/personality/samza.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=master
+#shellcheck disable=SC2034
+JIRA_ISSUE_RE='^SAMZA-[0-9]+$'
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/SAMZA/Contributor's+Corner"
+# shellcheck disable=SC2034
+BUILDTOOL=gradle
+#shellcheck disable=SC2034
+GITHUB_REPO="apache/samza"
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/tajo.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/tajo.sh b/dev-support/personality/tajo.sh
index 7e7ea97..209d75e 100755
--- a/dev-support/personality/tajo.sh
+++ b/dev-support/personality/tajo.sh
@@ -21,42 +21,4 @@ JIRA_ISSUE_RE='^TAJO-[0-9]+$'
#shellcheck disable=SC2034
GITHUB_REPO="apache/tajo"
#shellcheck disable=SC2034
-HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"
-
-function personality_modules
-{
- local repostatus=$1
- local testtype=$2
- local extra=""
-
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- case ${testtype} in
- mvninstall)
- extra="-DskipTests"
- if [[ ${repostatus} == branch ]]; then
- personality_enqueue_module . "${extra}"
- return
- fi
- return
- ;;
- asflicense)
- # this is very fast and provides the full path if we do it from
- # the root of the source
- personality_enqueue_module .
- return
- ;;
- unit)
- ;;
- *)
- extra="-DskipTests"
- ;;
- esac
-
- for module in ${CHANGED_MODULES}; do
- # shellcheck disable=SC2086
- personality_enqueue_module ${module} ${extra}
- done
-}
+HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/personality/tez.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/tez.sh b/dev-support/personality/tez.sh
index 9b45759..39f0b53 100755
--- a/dev-support/personality/tez.sh
+++ b/dev-support/personality/tez.sh
@@ -22,47 +22,3 @@ JIRA_ISSUE_RE='^TEZ-[0-9]+$'
GITHUB_REPO="apache/tez"
#shellcheck disable=SC2034
HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"
-
-function personality_modules
-{
- local repostatus=$1
- local testtype=$2
- local extra=""
-
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- case ${testtype} in
- mvninstall)
- extra="-DskipTests"
- if [[ ${repostatus} == branch ]]; then
- personality_enqueue_module . "${extra}"
- return
- fi
- return
- ;;
- asflicense)
- # this is very fast and provides the full path if we do it from
- # the root of the source
- personality_enqueue_module .
- return
- ;;
- unit)
- if [[ ${TEST_PARALLEL} == "true" ]] ; then
- extra="-Pparallel-tests"
- if [[ -n ${TEST_THREADS:-} ]]; then
- extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
- fi
- fi
- ;;
- *)
- extra="-DskipTests"
- ;;
- esac
-
- for module in ${CHANGED_MODULES}; do
- # shellcheck disable=SC2086
- personality_enqueue_module ${module} ${extra}
- done
-}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/ant.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/ant.sh b/dev-support/test-patch.d/ant.sh
new file mode 100755
index 0000000..8d1f364
--- /dev/null
+++ b/dev-support/test-patch.d/ant.sh
@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -z "${ANT_HOME:-}" ]]; then
+ ANT=ant
+else
+ ANT=${ANT_HOME}/bin/ant
+fi
+
+add_build_tool ant
+
+declare -a ANT_ARGS=("-noinput")
+
+function ant_usage
+{
+ echo "ant specific:"
+ echo "--ant-cmd=<cmd> The 'ant' command to use (default \${ANT_HOME}/bin/ant, or 'ant')"
+}
+
+function ant_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --ant-cmd=*)
+ ANT=${i#*=}
+ ;;
+ esac
+ done
+
+ # if we requested offline, pass that to ant
+ if [[ ${OFFLINE} == "true" ]]; then
+ ANT_ARGS=("${ANT_ARGS[@]}" -Doffline=)
+ fi
+}
+
+function ant_buildfile
+{
+ echo "build.xml"
+}
+
+function ant_executor
+{
+ echo "${ANT}" "${ANT_ARGS[@]}"
+}
+
+function ant_modules_worker
+{
+ declare repostatus=$1
+ declare tst=$2
+ shift 2
+
+ # shellcheck disable=SC2034
+ UNSUPPORTED_TEST=false
+
+ case ${tst} in
+ findbugs)
+ modules_workers "${repostatus}" findbugs findbugs
+ ;;
+ compile)
+ modules_workers "${repostatus}" compile
+ ;;
+ distclean)
+ modules_workers "${repostatus}" distclean clean
+ ;;
+ javadoc)
+ modules_workers "${repostatus}" javadoc clean javadoc
+ ;;
+ unit)
+ modules_workers "${repostatus}" unit
+ ;;
+ *)
+ # shellcheck disable=SC2034
+ UNSUPPORTED_TEST=true
+ if [[ ${repostatus} = patch ]]; then
+ add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin"
+ fi
+ yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}"
+ return 1
+ ;;
+ esac
+}
+
+function ant_javac_count_probs
+{
+ declare warningfile=$1
+ declare val1
+ declare val2
+
+ #shellcheck disable=SC2016
+ val1=$(${GREP} -E "\[javac\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ #shellcheck disable=SC2016
+ val2=$(${GREP} -E "\[javac\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ echo $((val1+val2))
+}
+
+## @description Helper for check_patch_javadoc
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function ant_javadoc_count_probs
+{
+ local warningfile=$1
+ local val1
+ local val2
+
+ #shellcheck disable=SC2016
+ val1=$(${GREP} -E "\[javadoc\] [0-9]+ errors?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ #shellcheck disable=SC2016
+ val2=$(${GREP} -E "\[javadoc\] [0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$2} END {print sum}')
+ echo $((val1+val2))
+}
+
+function ant_builtin_personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+
+ local module
+
+ yetus_debug "Using builtin personality_modules"
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module}
+ done
+}
+
+function ant_builtin_personality_file_tests
+{
+ local filename=$1
+
+ yetus_debug "Using builtin ant personality_file_tests"
+
+ if [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ src/test
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test javac
+ add_test unit
+ elif [[ ${filename} =~ build.xml
+ || ${filename} =~ ivy.xml
+ || ${filename} =~ \.java$
+ ]]; then
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test javac
+ add_test javadoc
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/asflicense.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/asflicense.sh b/dev-support/test-patch.d/asflicense.sh
index 27349bf..d839d5f 100755
--- a/dev-support/test-patch.d/asflicense.sh
+++ b/dev-support/test-patch.d/asflicense.sh
@@ -14,11 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-if [[ ${BUILDTOOL} == maven
- || ${BUILDTOOL} == ant ]]; then
- add_plugin asflicense
- add_test asflicense
-fi
+
+add_plugin asflicense
+add_test asflicense
## @description Verify all files have an Apache License
## @audience private
@@ -26,7 +24,7 @@ fi
## @replaceable no
## @return 0 on success
## @return 1 on failure
-function asflicense_postapply
+function asflicense_tests
{
local numpatch
@@ -36,12 +34,15 @@ function asflicense_postapply
personality_modules patch asflicense
case ${BUILDTOOL} in
- maven)
- modules_workers patch asflicense apache-rat:check
- ;;
ant)
modules_workers patch asflicense releaseaudit
;;
+ gradle)
+ modules_workers patch asflicense rat
+ ;;
+ maven)
+ modules_workers patch asflicense apache-rat:check
+ ;;
*)
return 0
;;
@@ -55,7 +56,9 @@ function asflicense_postapply
fi
#shellcheck disable=SC2038
- find "${BASEDIR}" -name rat.txt -o -name releaseaudit_report.txt \
+ find "${BASEDIR}" -name rat.txt \
+ -o -name releaseaudit_report.txt \
+ -o -name rat-report.txt \
| xargs cat > "${PATCH_DIR}/patch-asflicense.txt"
if [[ -s "${PATCH_DIR}/patch-asflicense.txt" ]] ; then
@@ -77,7 +80,7 @@ function asflicense_postapply
add_footer_table asflicense "@@BASE@@/patch-asflicense-problems.txt"
fi
else
- # if we're here, then maven actually failed
+ # if we're here, then build actually failed
modules_messages patch asflicense true
fi
return 1
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/author.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/author.sh b/dev-support/test-patch.d/author.sh
new file mode 100755
index 0000000..49824a5
--- /dev/null
+++ b/dev-support/test-patch.d/author.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin author
+
+## @description Check the current directory for @author tags
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function author_patchfile
+{
+ declare patchfile=$1
+ declare authorTags
+ # shellcheck disable=SC2155
+ declare -r appname=$(basename "${BASH_SOURCE-$0}")
+
+ big_console_header "Checking there are no @author tags in the patch."
+
+ start_clock
+
+ if [[ ${CHANGED_FILES} =~ ${appname} ]]; then
+ echo "Skipping @author checks as ${appname} has been patched."
+ add_vote_table 0 @author "Skipping @author checks as ${appname} has been patched."
+ return 0
+ fi
+
+ authorTags=$("${GREP}" -c -i '^[^-].*@author' "${patchfile}")
+ echo "There appear to be ${authorTags} @author tags in the patch."
+ if [[ ${authorTags} != 0 ]] ; then
+ add_vote_table -1 @author \
+ "The patch appears to contain ${authorTags} @author tags which the" \
+ " community has agreed to not allow in code contributions."
+ return 1
+ fi
+ add_vote_table +1 @author "The patch does not contain any @author tags."
+ return 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/builtin-bugsystem.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/builtin-bugsystem.sh b/dev-support/test-patch.d/builtin-bugsystem.sh
old mode 100644
new mode 100755
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/builtin-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/builtin-personality.sh b/dev-support/test-patch.d/builtin-personality.sh
index 4be3bfa..1f96e03 100755
--- a/dev-support/test-patch.d/builtin-personality.sh
+++ b/dev-support/test-patch.d/builtin-personality.sh
@@ -14,148 +14,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-function builtin_personality_modules
-{
- local repostatus=$1
- local testtype=$2
-
- local module
-
- yetus_debug "Using builtin personality_modules"
- yetus_debug "Personality: ${repostatus} ${testtype}"
-
- clear_personality_queue
-
- # this always makes sure the local repo has a fresh
- # copy of everything per pom rules.
- if [[ ${repostatus} == branch
- && ${testtype} == mvninstall ]];then
- personality_enqueue_module .
- return
- fi
-
- for module in ${CHANGED_MODULES}; do
- # shellcheck disable=SC2086
- personality_enqueue_module ${module}
- done
-}
-
function personality_modules
{
- builtin_personality_modules "$@"
-}
-
-function builtin_mvn_personality_file_tests
-{
- local filename=$1
-
- yetus_debug "Using builtin mvn personality_file_tests"
-
- if [[ ${filename} =~ src/main/webapp ]]; then
- yetus_debug "tests/webapp: ${filename}"
- elif [[ ${filename} =~ \.sh
- || ${filename} =~ \.cmd
- || ${filename} =~ src/main/scripts
- || ${filename} =~ src/test/scripts
- ]]; then
- yetus_debug "tests/shell: ${filename}"
- elif [[ ${filename} =~ \.md$
- || ${filename} =~ \.md\.vm$
- || ${filename} =~ src/site
- || ${filename} =~ src/main/docs
- ]]; then
- yetus_debug "tests/site: ${filename}"
- add_test site
- elif [[ ${filename} =~ \.c$
- || ${filename} =~ \.cc$
- || ${filename} =~ \.h$
- || ${filename} =~ \.hh$
- || ${filename} =~ \.proto$
- || ${filename} =~ \.cmake$
- || ${filename} =~ CMakeLists.txt
- ]]; then
- yetus_debug "tests/units: ${filename}"
- add_test cc
- add_test unit
- elif [[ ${filename} =~ \.scala$ ]]; then
- add_test javac
- add_test unit
- add_test mvninstall
- elif [[ ${filename} =~ build.xml$
- || ${filename} =~ pom.xml$
- || ${filename} =~ \.java$
- || ${filename} =~ src/main
- ]]; then
- yetus_debug "tests/javadoc+units: ${filename}"
- add_test javac
- add_test javadoc
- add_test mvninstall
- add_test unit
- fi
-
- if [[ ${filename} =~ src/test ]]; then
- yetus_debug "tests"
- add_test unit
- fi
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test findbugs
- fi
-}
-
-function builtin_ant_personality_file_tests
-{
- local filename=$1
-
- yetus_debug "Using builtin ant personality_file_tests"
-
- if [[ ${filename} =~ \.sh
- || ${filename} =~ \.cmd
- ]]; then
- yetus_debug "tests/shell: ${filename}"
- elif [[ ${filename} =~ \.c$
- || ${filename} =~ \.cc$
- || ${filename} =~ \.h$
- || ${filename} =~ \.hh$
- || ${filename} =~ \.proto$
- || ${filename} =~ src/test
- || ${filename} =~ \.cmake$
- || ${filename} =~ CMakeLists.txt
- ]]; then
- yetus_debug "tests/units: ${filename}"
- add_test javac
- add_test unit
- elif [[ ${filename} =~ build.xml
- || ${filename} =~ ivy.xml
- || ${filename} =~ \.java$
- ]]; then
- yetus_debug "tests/javadoc+units: ${filename}"
- add_test javac
- add_test javadoc
- add_test unit
- fi
-
- if [[ ${filename} =~ \.java$ ]]; then
- add_test findbugs
- fi
-}
-
-function builtin_personality_file_tests
-{
- case ${BUILDTOOL} in
- maven)
- builtin_mvn_personality_file_tests "$@"
- ;;
- ant)
- builtin_ant_personality_file_tests "$@"
- ;;
- *)
- return 1
- ;;
- esac
+ "${BUILDTOOL}_builtin_personality_modules" "$@"
}
function personality_file_tests
{
- builtin_personality_file_tests "$@"
+ "${BUILDTOOL}_builtin_personality_file_tests" "$@"
}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/checkstyle.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/checkstyle.sh b/dev-support/test-patch.d/checkstyle.sh
index 5538790..bc47b57 100755
--- a/dev-support/test-patch.d/checkstyle.sh
+++ b/dev-support/test-patch.d/checkstyle.sh
@@ -60,21 +60,28 @@ function checkstyle_runner
modulesuffix=$(basename "${MODULE[${i}]}")
output="${PATCH_DIR}/${repostatus}-checkstyle-${fn}.txt"
logfile="${PATCH_DIR}/maven-${repostatus}-checkstyle-${fn}.txt"
- pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null
+
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ pushd "${BASEDIR}/${MODULE[${i}]}" >/dev/null
+ fi
case ${BUILDTOOL} in
- maven)
- cmd="${MVN} ${MAVEN_ARGS[*]} \
- checkstyle:checkstyle \
- -Dcheckstyle.consoleOutput=true \
- ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch"
- ;;
ant)
cmd="${ANT} \
-Dcheckstyle.consoleOutput=true \
${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} \
${ANT_ARGS[*]} checkstyle"
;;
+ maven)
+ cmd="${MAVEN} ${MAVEN_ARGS[*]} \
+ checkstyle:checkstyle \
+ -Dcheckstyle.consoleOutput=true \
+ ${MODULEEXTRAPARAM[${i}]//@@@MODULEFN@@@/${fn}} -Ptest-patch"
+ ;;
+ *)
+ UNSUPPORTED_TEST=true
+ return 0
+ ;;
esac
#shellcheck disable=SC2086
@@ -101,8 +108,10 @@ function checkstyle_runner
done
rm "${tmp}" 2>/dev/null
- # shellcheck disable=SC2086
- popd >/dev/null
+
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ popd >/dev/null
+ fi
((i=i+1))
done
@@ -114,20 +123,30 @@ function checkstyle_runner
return 0
}
-function checkstyle_preapply
+function checkstyle_postcompile
{
- local result
+ declare repostatus=$1
- big_console_header "${PATCH_BRANCH} checkstyle"
+ if [[ "${repostatus}" = branch ]]; then
+ checkstyle_preapply
+ else
+ checkstyle_postapply
+ fi
+}
- start_clock
+function checkstyle_preapply
+{
+ local result
verify_needed_test checkstyle
if [[ $? == 0 ]]; then
- echo "Patch does not need checkstyle testing."
return 0
fi
+ big_console_header "${PATCH_BRANCH} checkstyle"
+
+ start_clock
+
personality_modules branch checkstyle
checkstyle_runner branch
result=$?
@@ -151,20 +170,22 @@ function checkstyle_postapply
local numpostpatch=0
local diffpostpatch=0
- big_console_header "Patch checkstyle plugin"
-
- start_clock
-
verify_needed_test checkstyle
if [[ $? == 0 ]]; then
- echo "Patch does not need checkstyle testing."
return 0
fi
+ big_console_header "Patch checkstyle plugin"
+
+ start_clock
+
personality_modules patch checkstyle
checkstyle_runner patch
result=$?
+ if [[ ${UNSUPPORTED_TEST} = true ]]; then
+ return 0
+ fi
# add our previous elapsed to our new timer
# by setting the clock back
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/findbugs.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/findbugs.sh b/dev-support/test-patch.d/findbugs.sh
index df479af..cc251bb 100755
--- a/dev-support/test-patch.d/findbugs.sh
+++ b/dev-support/test-patch.d/findbugs.sh
@@ -89,14 +89,11 @@ function findbugs_runner
local savestop
personality_modules "${name}" findbugs
- case ${BUILDTOOL} in
- maven)
- modules_workers "${name}" findbugs test-compile findbugs:findbugs
- ;;
- ant)
- modules_workers "${name}" findbugs findbugs
- ;;
- esac
+ "${BUILDTOOL}_modules_worker" "${name}" findbugs
+
+ if [[ ${UNSUPPORTED_TEST} = true ]]; then
+ return 0
+ fi
#shellcheck disable=SC2153
until [[ ${i} -eq ${#MODULE[@]} ]]; do
@@ -119,6 +116,7 @@ function findbugs_runner
;;
esac
+
if [[ ! -f ${file} ]]; then
module_status ${i} -1 "" "${name}/${module} no findbugs output file (${file})"
((i=i+1))
@@ -185,12 +183,9 @@ function findbugs_preapply
local module_findbugs_warnings
local result=0
- big_console_header "Pre-patch findbugs detection"
-
verify_needed_test findbugs
if [[ $? == 0 ]]; then
- echo "Patch does not appear to need findbugs tests."
return 0
fi
@@ -199,9 +194,15 @@ function findbugs_preapply
return 1
fi
+ big_console_header "Pre-patch findbugs detection"
+
findbugs_runner branch
result=$?
+ if [[ ${UNSUPPORTED_TEST} = true ]]; then
+ return 0
+ fi
+
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
until [[ $i -eq ${#MODULE[@]} ]]; do
if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
@@ -262,15 +263,14 @@ function findbugs_postinstall
local result=0
local savestop
- big_console_header "Patch findbugs detection"
-
verify_needed_test findbugs
if [[ $? == 0 ]]; then
- echo "Patch does not appear to need findbugs tests."
return 0
fi
+ big_console_header "Patch findbugs detection"
+
findbugs_is_installed
if [[ $? != 0 ]]; then
return 1
@@ -278,6 +278,10 @@ function findbugs_postinstall
findbugs_runner patch
+ if [[ ${UNSUPPORTED_TEST} = true ]]; then
+ return 0
+ fi
+
until [[ $i -eq ${#MODULE[@]} ]]; do
if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
((result=result+1))
@@ -287,7 +291,11 @@ function findbugs_postinstall
start_clock
offset_clock "${MODULE_STATUS_TIMER[${i}]}"
module="${MODULE[${i}]}"
- pushd "${module}" >/dev/null
+
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ pushd "${module}" >/dev/null
+ fi
+
fn=$(module_file_fragment "${module}")
combined_xml="${PATCH_DIR}/combined-findbugs-${fn}.xml"
@@ -306,7 +314,9 @@ function findbugs_postinstall
"${branchxml}" \
"${patchxml}"
if [[ $? != 0 ]]; then
- popd >/dev/null
+ if [[ ${BUILDTOOLCWD} == true ]]; then
+ popd >/dev/null
+ fi
module_status ${i} -1 "" "${module} cannot run computeBugHistory from findbugs"
((result=result+1))
savestop=$(stop_clock)
@@ -379,3 +389,14 @@ function findbugs_postinstall
fi
return 0
}
+
+function findbugs_rebuild
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ findbugs_preapply
+ else
+ findbugs_postinstall
+ fi
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/gradle.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/gradle.sh b/dev-support/test-patch.d/gradle.sh
new file mode 100755
index 0000000..353eb91
--- /dev/null
+++ b/dev-support/test-patch.d/gradle.sh
@@ -0,0 +1,248 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_build_tool gradle
+
+declare -a GRADLE_ARGS=()
+
+function gradle_usage
+{
+ echo "gradle specific:"
+ echo "--gradle-cmd=<cmd> The 'gradle' command to use (default 'gradle')"
+ echo "--gradlew-cmd=<cmd> The 'gradle' command to use (default 'basedir/gradlew')"
+}
+
+function gradle_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --gradle-cmd=*)
+ GRADLE=${i#*=}
+ ;;
+ --gradlew-cmd=*)
+ GRADLEW=${i#*=}
+ ;;
+ esac
+ done
+
+ # if we requested offline, pass that to mvn
+ if [[ ${OFFLINE} == "true" ]]; then
+ GRADLE_ARGS=("${GRADLE_ARGS[@]}" --offline)
+ fi
+
+ GRADLE=${GRADLE:-gradle}
+ GRADLEW=${GRADLEW:-"${BASEDIR}/gradlew"}
+}
+
+function gradle_initialize
+{
+ if [[ "${BUILDTOOL}" = gradle ]]; then
+ # shellcheck disable=SC2034
+ BUILDTOOLCWD=false
+ fi
+}
+
+function gradle_buildfile
+{
+ echo "gradlew"
+}
+
+function gradle_executor
+{
+ echo "${GRADLEW}" "${GRADLE_ARGS[@]}"
+}
+
+## @description Bootstrap gradle
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function gradle_precompile
+{
+ declare repostatus=$1
+ declare result=0
+
+ if [[ ${BUILDTOOL} != gradle ]]; then
+ return 0
+ fi
+
+ if [[ "${repostatus}" = branch ]]; then
+ # shellcheck disable=SC2153
+ big_console_header "${PATCH_BRANCH} gradle bootstrap"
+ else
+ big_console_header "Patch gradle bootstrap"
+ fi
+
+ personality_modules "${repostatus}" gradleboot
+
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/${repostatus}-gradle-bootstrap.txt" gradle -b bootstrap.gradle
+ popd >/dev/null
+
+ modules_workers "${repostatus}" gradleboot
+ result=$?
+ modules_messages "${repostatus}" gradleboot true
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+function gradle_scalac_count_probs
+{
+ local warningfile=$1
+
+ #shellcheck disable=SC2016,SC2046
+ ${GREP} "^/.*.scala:[0-9]*:" "${warningfile}" | wc -l
+}
+
+function gradle_javac_count_probs
+{
+ echo 0
+}
+
+function gradle_javadoc_count_probs
+{
+ echo 0
+}
+
+## @description Helper for check_patch_javadoc
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function gradle_scaladoc_count_probs
+{
+ local warningfile=$1
+
+ #shellcheck disable=SC2016,SC2046
+ ${GREP} "^\[ant:scaladoc\]" "${warningfile}" | wc -l
+}
+
+function gradle_modules_worker
+{
+ declare repostatus=$1
+ declare tst=$2
+ shift 2
+
+ # shellcheck disable=SC2034
+ UNSUPPORTED_TEST=false
+
+ case ${tst} in
+ checkstyle)
+ modules_workers "${repostatus}" "${tst}" checkstyleMain checkstyleTest
+ ;;
+ compile)
+ modules_workers "${repostatus}" "${tst}"
+ ;;
+ distclean)
+ modules_workers "${repostatus}" clean
+ ;;
+ javadoc)
+ modules_workers "${repostatus}" "${tst}" javadoc
+ ;;
+ scaladoc)
+ modules_workers "${repostatus}" "${tst}" scaladoc
+ ;;
+ unit)
+ modules_workers "${repostatus}" "${tst}" test
+ ;;
+ *)
+ # shellcheck disable=SC2034
+ UNSUPPORTED_TEST=true
+ if [[ ${repostatus} = patch ]]; then
+ add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin"
+ fi
+ yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}"
+ return 1
+ ;;
+ esac
+}
+
+function gradle_builtin_personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+
+ local module
+
+ yetus_debug "Using builtin personality_modules"
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module}
+ done
+}
+
+function gradle_builtin_personality_file_tests
+{
+ local filename=$1
+
+ yetus_debug "Using builtin gradle personality_file_tests"
+
+ if [[ ${filename} =~ src/main/webapp ]]; then
+ yetus_debug "tests/webapp: ${filename}"
+ elif [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ || ${filename} =~ src/main/scripts
+ || ${filename} =~ src/test/scripts
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test cc
+ add_test unit
+ elif [[ ${filename} =~ \.scala$ ]]; then
+ add_test scalac
+ add_test scaladoc
+ add_test unit
+ elif [[ ${filename} =~ build.xml$
+ || ${filename} =~ pom.xml$
+ || ${filename} =~ \.java$
+ ]]; then
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test javac
+ add_test javadoc
+ add_test unit
+ elif [[ ${filename} =~ src/main ]]; then
+ yetus_debug "tests/generic+units: ${filename}"
+ add_test compile
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ src/test ]]; then
+ yetus_debug "tests"
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/java.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/java.sh b/dev-support/test-patch.d/java.sh
new file mode 100755
index 0000000..b07c67b
--- /dev/null
+++ b/dev-support/test-patch.d/java.sh
@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_plugin javac
+add_plugin javadoc
+
+function javac_initialize
+{
+ local i
+ local jdkdir
+ local tmplist
+
+ if [[ -z ${JAVA_HOME:-} ]]; then
+ case ${OSTYPE} in
+ Darwin)
+ if [[ -z "${JAVA_HOME}" ]]; then
+ if [[ -x /usr/libexec/java_home ]]; then
+ JAVA_HOME="$(/usr/libexec/java_home)"
+ export JAVA_HOME
+ else
+ export JAVA_HOME=/Library/Java/Home
+ fi
+ fi
+ ;;
+ *)
+ yetus_error "WARNING: JAVA_HOME not defined. Disabling java tests."
+ delete_test javac
+ delete_test javadoc
+ return 1
+ ;;
+ esac
+ fi
+
+ JAVA_HOME=$(cd -P -- "${JAVA_HOME}" >/dev/null && pwd -P)
+
+ for i in ${JDK_DIR_LIST}; do
+ jdkdir=$(cd -P -- "${i}" >/dev/null && pwd -P)
+ if [[ ${jdkdir} != "${JAVA_HOME}" ]]; then
+ tmplist="${tmplist} ${jdkdir}"
+ fi
+ done
+
+ JDK_DIR_LIST="${tmplist} ${JAVA_HOME}"
+ JDK_DIR_LIST=${JDK_DIR_LIST/ }
+}
+
+## @description Verify that ${JAVA_HOME} is defined
+## @audience public
+## @stability stable
+## @replaceable no
+## @return 1 - no JAVA_HOME
+## @return 0 - JAVA_HOME defined
+function javac_precheck
+{
+ declare javaversion
+ declare listofjdks
+ declare i
+
+ start_clock
+
+ if [[ -z ${JAVA_HOME:-} ]]; then
+ yetus_error "ERROR: JAVA_HOME is not defined."
+ add_vote_table -1 pre-patch "JAVA_HOME is not defined."
+ return 1
+ fi
+
+ javaversion=$(report_jvm_version "${JAVA_HOME}")
+ add_footer_table "Default Java" "${javaversion}"
+
+ if [[ -n ${JDK_DIR_LIST}
+ && ${JDK_DIR_LIST} != "${JAVA_HOME}" ]]; then
+ for i in ${JDK_DIR_LIST}; do
+ javaversion=$(report_jvm_version "${i}")
+ listofjdks="${listofjdks} ${i}:${javaversion}"
+ done
+ add_footer_table "Multi-JDK versions" "${listofjdks}"
+ fi
+ return 0
+}
+
+function javac_filefilter
+{
+ declare filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ yetus_debug "tests/javac: ${filename}"
+ add_test javac
+ add_test compile
+ fi
+}
+
+function javadoc_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ yetus_debug "tests/javadoc: ${filename}"
+ add_test javadoc
+ fi
+}
+
+## @description
+## @audience private
+## @stability stable
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function javac_compile
+{
+ declare codebase=$1
+ declare multijdkmode=$2
+
+ verify_needed_test javac
+ if [[ $? = 0 ]]; then
+ return 0
+ fi
+
+ if [[ ${codebase} = patch ]]; then
+ yetus_debug "javac: calling generic_postlog_compare compile javac ${multijdkmode}"
+ generic_postlog_compare compile javac "${multijdkmode}"
+ fi
+}
+
+## @description Count and compare the number of JavaDoc warnings pre- and post- patch
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function javadoc_rebuild
+{
+ declare codebase=$1
+ declare multijdkmode
+
+ verify_multijdk_test javadoc
+ if [[ $? == 1 ]]; then
+ multijdkmode=true
+ fi
+
+ if [[ "${codebase}" = branch ]]; then
+ generic_pre_handler javadoc "${multijdkmode}"
+ else
+ generic_post_handler javadoc javadoc "${multijdkmode}" true
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/jira.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/jira.sh b/dev-support/test-patch.d/jira.sh
index fc649e7..59baf0f 100755
--- a/dev-support/test-patch.d/jira.sh
+++ b/dev-support/test-patch.d/jira.sh
@@ -248,7 +248,7 @@ function jira_determine_branch
count="${PATCH_BRANCH//[^.]}"
total=${#count}
((total = total + 1 ))
- until [[ ${total} -eq 1 ]]; do
+ until [[ ${total} -lt 1 ]]; do
PATCH_BRANCH=$(echo "${patchnamechunk}" | cut -f3- -d- | cut -f1-${total} -d. )
yetus_debug "Determine branch: ISSUE-branch[.##] = ${PATCH_BRANCH}"
((total=total-1))
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/maven.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/maven.sh b/dev-support/test-patch.d/maven.sh
new file mode 100755
index 0000000..94ac383
--- /dev/null
+++ b/dev-support/test-patch.d/maven.sh
@@ -0,0 +1,328 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare -a MAVEN_ARGS=("--batch-mode")
+
+if [[ -z "${MAVEN_HOME:-}" ]]; then
+ MAVEN=mvn
+else
+ MAVEN=${MAVEN_HOME}/bin/mvn
+fi
+
+add_plugin mvnsite
+add_plugin mvneclipse
+add_build_tool maven
+
+function maven_usage
+{
+ echo "maven specific:"
+ echo "--mvn-cmd=<cmd> The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn, or 'mvn')"
+}
+
+function maven_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --mvn-cmd=*)
+ MAVEN=${i#*=}
+ ;;
+ esac
+ done
+
+ if [[ ${OFFLINE} == "true" ]]; then
+ MAVEN_ARGS=("${MAVEN_ARGS[@]}" --offline)
+ fi
+}
+
+function maven_buildfile
+{
+ echo "pom.xml"
+}
+
+function maven_executor
+{
+ echo "${MAVEN}" "${MAVEN_ARGS[@]}"
+}
+
+function mvnsite_filefilter
+{
+ local filename=$1
+
+ if [[ ${BUILDTOOL} = maven ]]; then
+ if [[ ${filename} =~ src/site ]]; then
+ yetus_debug "tests/mvnsite: ${filename}"
+ add_test mvnsite
+ fi
+ fi
+}
+
+function maven_modules_worker
+{
+ declare repostatus=$1
+ declare tst=$2
+
+ # shellcheck disable=SC2034
+ UNSUPPORTED_TEST=false
+
+ case ${tst} in
+ findbugs)
+ modules_workers "${repostatus}" findbugs test-compile findbugs:findbugs -DskipTests=true
+ ;;
+ compile)
+ modules_workers "${repostatus}" compile clean test-compile -DskipTests=true
+ ;;
+ distclean)
+ modules_workers "${repostatus}" distclean clean -DskipTests=true
+ ;;
+ javadoc)
+ modules_workers "${repostatus}" javadoc clean javadoc:javadoc -DskipTests=true
+ ;;
+ scaladoc)
+ modules_workers "${repostatus}" scaladoc clean scala:doc -DskipTests=true
+ ;;
+ unit)
+ modules_workers "${repostatus}" unit clean test -fae
+ ;;
+ *)
+ # shellcheck disable=SC2034
+ UNSUPPORTED_TEST=true
+ if [[ ${repostatus} = patch ]]; then
+ add_footer_table "${tst}" "not supported by the ${BUILDTOOL} plugin"
+ fi
+ yetus_error "WARNING: ${tst} is unsupported by ${BUILDTOOL}"
+ return 1
+ ;;
+ esac
+}
+
+function maven_javac_count_probs
+{
+ local warningfile=$1
+
+ #shellcheck disable=SC2016,SC2046
+ ${GREP} '\[WARNING\]' "${warningfile}" | ${AWK} '{sum+=1} END {print sum}'
+}
+
+function maven_scalac_count_probs
+{
+ echo 0
+}
+
+## @description Helper for check_patch_javadoc
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function maven_javadoc_count_probs
+{
+ local warningfile=$1
+
+ #shellcheck disable=SC2016,SC2046
+ ${GREP} -E "^[0-9]+ warnings?$" "${warningfile}" | ${AWK} '{sum+=$1} END {print sum}'
+}
+
+function maven_builtin_personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+
+ local module
+
+ yetus_debug "Using builtin personality_modules"
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ # this always makes sure the local repo has a fresh
+ # copy of everything per pom rules.
+ if [[ ${repostatus} == branch
+ && ${testtype} == mvninstall ]];then
+ personality_enqueue_module .
+ return
+ fi
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module}
+ done
+}
+
+function maven_builtin_personality_file_tests
+{
+ local filename=$1
+
+ yetus_debug "Using builtin mvn personality_file_tests"
+
+ if [[ ${filename} =~ src/main/webapp ]]; then
+ yetus_debug "tests/webapp: ${filename}"
+ elif [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ || ${filename} =~ src/main/scripts
+ || ${filename} =~ src/test/scripts
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test cc
+ add_test unit
+ elif [[ ${filename} =~ \.scala$
+ || ${filename} =~ src/scala ]]; then
+ add_test scalac
+ add_test scaladoc
+ add_test unit
+ elif [[ ${filename} =~ build.xml$
+ || ${filename} =~ pom.xml$
+ || ${filename} =~ \.java$
+ || ${filename} =~ src/main
+ ]]; then
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test javac
+ add_test javadoc
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ src/test ]]; then
+ yetus_debug "tests"
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
+
+## @description Confirm site pre-patch
+## @audience private
+## @stability stable
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function mvnsite_postcompile
+{
+ declare repostatus=$1
+ declare result=0
+
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ verify_needed_test mvnsite
+ if [[ $? == 0 ]];then
+ return 0
+ fi
+
+ if [[ "${repostatus}" = branch ]]; then
+ big_console_header "Pre-patch ${PATCH_BRANCH} maven site verification"
+ else
+ big_console_header "Patch maven site verification"
+ fi
+
+
+ personality_modules "${repostatus}" mvnsite
+ modules_workers "${repostatus}" mvnsite clean site site:stage
+ result=$?
+ modules_messages "${repostatus}" mvnsite true
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Make sure Maven's eclipse generation works.
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function mvneclipse_postcompile
+{
+ declare repostatus=$1
+ declare result=0
+
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ verify_needed_test javac
+ if [[ $? == 0 ]]; then
+ return 0
+ fi
+
+ if [[ "${repostatus}" = branch ]]; then
+ big_console_header "Pre-patch ${PATCH_BRANCH} maven eclipse verification"
+ else
+ big_console_header "Patch maven eclipse verification"
+ fi
+
+ personality_modules "${repostatus}" mvneclipse
+ modules_workers "${repostatus}" mvneclipse eclipse:clean eclipse:eclipse
+ result=$?
+ modules_messages "${repostatus}" mvneclipse true
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Verify mvn install works
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function maven_precompile
+{
+ declare repostatus=$1
+ declare result=0
+
+ if [[ ${BUILDTOOL} != maven ]]; then
+ return 0
+ fi
+
+ verify_needed_test javadoc
+ result=$?
+
+ verify_needed_test javac
+ ((result = result + $? ))
+ if [[ ${result} == 0 ]]; then
+ return 0
+ fi
+
+ if [[ "${repostatus}" = branch ]]; then
+ big_console_header "Pre-patch ${PATCH_BRANCH} maven install"
+ else
+ big_console_header "Patch maven install"
+ fi
+
+ personality_modules "${repostatus}" mvninstall
+ modules_workers "${repostatus}" mvninstall -fae clean install -DskipTests=true -Dmaven.javadoc.skip=true
+ result=$?
+ modules_messages "${repostatus}" mvninstall true
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/perlcritic.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/perlcritic.sh b/dev-support/test-patch.d/perlcritic.sh
index 1cec3f3..c3ef6c9 100755
--- a/dev-support/test-patch.d/perlcritic.sh
+++ b/dev-support/test-patch.d/perlcritic.sh
@@ -138,3 +138,14 @@ function perlcritic_postapply
add_vote_table +1 perlcritic "There were no new perlcritic issues."
return 0
}
+
+function perlcritic_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ perlcritic_preapply
+ else
+ perlcritic_postapply
+ fi
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/pylint.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/pylint.sh b/dev-support/test-patch.d/pylint.sh
index 6fa576e..4519031 100755
--- a/dev-support/test-patch.d/pylint.sh
+++ b/dev-support/test-patch.d/pylint.sh
@@ -166,3 +166,14 @@ function pylint_postapply
add_vote_table +1 pylint "There were no new pylint issues."
return 0
}
+
+function pylint_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ pylint_preapply
+ else
+ pylint_postapply
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/rubocop.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/rubocop.sh b/dev-support/test-patch.d/rubocop.sh
index 091a2b1..0190289 100755
--- a/dev-support/test-patch.d/rubocop.sh
+++ b/dev-support/test-patch.d/rubocop.sh
@@ -138,3 +138,14 @@ function rubocop_postapply
add_vote_table +1 rubocop "There were no new rubocop issues."
return 0
}
+
+function rubocop_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ rubocop_preapply
+ else
+ rubocop_postapply
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/02226269/dev-support/test-patch.d/ruby-lint.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/ruby-lint.sh b/dev-support/test-patch.d/ruby-lint.sh
index 35d9604..86b7701 100755
--- a/dev-support/test-patch.d/ruby-lint.sh
+++ b/dev-support/test-patch.d/ruby-lint.sh
@@ -138,3 +138,14 @@ function ruby_lint_postapply
add_vote_table +1 ruby-lint "There were no new ruby-lint issues."
return 0
}
+
+function ruby_lint_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ ruby_lint_preapply
+ else
+ ruby_lint_postapply
+ fi
+}
[16/50] [abbrv] yetus git commit: Convert 'unit' to 'junit' (aw)
Posted by bu...@apache.org.
Convert 'unit' to 'junit' (aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/11ad56e2
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/11ad56e2
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/11ad56e2
Branch: refs/heads/master
Commit: 11ad56e2eb41bbc5e90afc9c8107e57691339a4f
Parents: a627ff0
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 3 19:40:51 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 3 19:40:51 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.d/junit.sh | 68 ++++++++++++++++++++++++++++++++++
dev-support/test-patch.sh | 53 ++++++++++----------------
2 files changed, 88 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/11ad56e2/dev-support/test-patch.d/junit.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/junit.sh b/dev-support/test-patch.d/junit.sh
new file mode 100755
index 0000000..4393511
--- /dev/null
+++ b/dev-support/test-patch.d/junit.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_test_format junit
+
+JUNIT_TEST_TIMEOUTS=""
+JUNIT_FAILED_TESTS=""
+
+function junit_process_tests
+{
+ # shellcheck disable=SC2034
+ declare module=$1
+ declare buildlogfile=$2
+ declare result=0
+ declare module_test_timeouts
+ declare module_failed_tests
+
+ # shellcheck disable=SC2016
+ module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${buildlogfile}")
+ if [[ -n "${module_test_timeouts}" ]] ; then
+ JUNIT_TEST_TIMEOUTS="${JUNIT_TEST_TIMEOUTS} ${module_test_timeouts}"
+ ((result=result+1))
+ fi
+
+ #shellcheck disable=SC2026,SC2038,SC2016
+ module_failed_tests=$(find . -name 'TEST*.xml'\
+ | xargs "${GREP}" -l -E "<failure|<error"\
+ | ${AWK} -F/ '{sub("TEST-org.apache.",""); sub(".xml",""); print $NF}')
+
+ if [[ -n "${module_failed_tests}" ]] ; then
+ JUNIT_FAILED_TESTS="${JUNIT_FAILED_TESTS} ${module_failed_tests}"
+ ((result=result+1))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+function junit_finalize_results
+{
+ declare jdk=$1
+
+ if [[ -n "${JUNIT_FAILED_TESTS}" ]] ; then
+ # shellcheck disable=SC2086
+ populate_test_table "${jdk}Failed junit tests" ${JUNIT_FAILED_TESTS}
+ JUNIT_FAILED_TESTS=""
+ fi
+ if [[ -n "${JUNIT_TEST_TIMEOUTS}" ]] ; then
+ # shellcheck disable=SC2086
+ populate_test_table "${jdk}Timed out junit tests" ${JUNIT_TEST_TIMEOUTS}
+ JUNIT_TEST_TIMEOUTS=""
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/11ad56e2/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 1f1f88e..1faf99f 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -102,6 +102,7 @@ function setup_defaults
OSTYPE=$(uname -s)
BUILDTOOL=maven
BUGSYSTEM=jira
+ TESTFORMATS=""
JDK_TEST_LIST="javac javadoc unit"
GITDIFFLINES="${PATCH_DIR}/gitdifflines.txt"
GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
@@ -774,7 +775,7 @@ function testpatch_usage
importplugins
- for plugin in ${PLUGINS} ${BUGSYSTEMS}; do
+ for plugin in ${PLUGINS} ${BUGSYSTEMS} ${TESTFORMATS}; do
if declare -f ${plugin}_usage >/dev/null 2>&1; then
echo
"${plugin}_usage"
@@ -2940,10 +2941,7 @@ function populate_test_table
function check_unittests
{
local i
- local failed_tests=""
- local test_timeouts=""
local test_logfile
- local module_test_timeouts=""
local result=0
local -r savejavahome=${JAVA_HOME}
local multijdkmode=false
@@ -3001,40 +2999,20 @@ function check_unittests
fn="${fn}${jdk}"
test_logfile="${PATCH_DIR}/patch-unit-${fn}.txt"
- # shellcheck disable=2016
- module_test_timeouts=$(${AWK} '/^Running / { array[$NF] = 1 } /^Tests run: .* in / { delete array[$NF] } END { for (x in array) { print x } }' "${test_logfile}")
- if [[ -n "${module_test_timeouts}" ]] ; then
- test_timeouts="${test_timeouts} ${module_test_timeouts}"
- ((result=result+1))
- fi
-
pushd "${MODULE[${i}]}" >/dev/null
- #shellcheck disable=SC2026,SC2038,SC2016
- module_failed_tests=$(find . -name 'TEST*.xml'\
- | xargs "${GREP}" -l -E "<failure|<error"\
- | ${AWK} -F/ '{sub("TEST-org.apache.",""); sub(".xml",""); print $NF}')
- popd >/dev/null
+ for j in ${TESTSYSTEMS}; do
+ if declare -f ${j}_process_tests; then
+ "${j}_process_tests" "${module}" "${test_logfile}"
+ ((results=results+$?))
+ fi
+ done
- if [[ -n "${module_failed_tests}" ]] ; then
- failed_tests="${failed_tests} ${module_failed_tests}"
- ((result=result+1))
- fi
+ popd >/dev/null
((i=i+1))
done
- if [[ -n "${failed_tests}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "${statusjdk}Failed unit tests" ${failed_tests}
- failed_tests=""
- fi
- if [[ -n "${test_timeouts}" ]] ; then
- # shellcheck disable=SC2086
- populate_test_table "${statusjdk}Timed out tests" ${test_timeouts}
- test_timeouts=""
- fi
-
done
JAVA_HOME=${savejavahome}
@@ -3433,7 +3411,7 @@ function importplugins
## @replaceable no
function parse_args_plugins
{
- for plugin in ${PLUGINS} ${BUGSYSTEMS}; do
+ for plugin in ${PLUGINS} ${BUGSYSTEMS} ${TESTFORMATS}; do
if declare -f ${plugin}_parse_args >/dev/null 2>&1; then
yetus_debug "Running ${plugin}_parse_args"
#shellcheck disable=SC2086
@@ -3452,7 +3430,7 @@ function add_plugin
PLUGINS="${PLUGINS} $1"
}
-## @description Register test-patch.d plugins
+## @description Register test-patch.d bugsystems
## @audience public
## @stability stable
## @replaceable no
@@ -3461,6 +3439,15 @@ function add_bugsystem
BUGSYSTEMS="${BUGSYSTEMS} $1"
}
+## @description Register test-patch.d test output formats
+## @audience public
+## @stability stable
+## @replaceable no
+function add_test_format
+{
+ TESTFORMATS="${TESTFORMATS} $1"
+}
+
## @description Calculate the differences between the specified files
## @description and output it to stdout.
## @audience public
[37/50] [abbrv] yetus git commit: Revert "HADOOP-12233. if
CHANGED_FILES is corrupt,
find_changed_modules never returns (Kengo Seki via aw)"
Posted by bu...@apache.org.
Revert "HADOOP-12233. if CHANGED_FILES is corrupt, find_changed_modules never returns (Kengo Seki via aw)"
This reverts commit a32b5b01b709ae11dc80c9f972b2e7b84bbce904.
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/3ee84912
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/3ee84912
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/3ee84912
Branch: refs/heads/master
Commit: 3ee849126c5fd3838a8632dc41266fef7047888a
Parents: 1253205
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Aug 26 09:02:09 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Aug 26 09:02:09 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 14 --------------
1 file changed, 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/3ee84912/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 9f08ad3..4d4b63f 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1107,7 +1107,6 @@ function find_buildfile_dir
{
local buildfile=$1
local dir=$2
- local d
yetus_debug "Find ${buildfile} dir for: ${dir}"
@@ -1120,12 +1119,6 @@ function find_buildfile_dir
yetus_debug "ERROR: ${buildfile} is not found."
return 1
else
- d=$(cd -P -- "$(dirname -- "${dir}")" >/dev/null && pwd -P)
- relative_dir "${d}" >/dev/null
- if [[ $? == 1 ]]; then
- yetus_debug "ERROR: ${dir} is not in ${BASEDIR}."
- return 1
- fi
dir=$(dirname "${dir}")
fi
done
@@ -1158,7 +1151,6 @@ function find_changed_files
function module_skipdir
{
local dir=${1}
- local d
local i
yetus_debug "Checking skipdirs for ${dir}"
@@ -1178,12 +1170,6 @@ function module_skipdir
if [[ ${dir} == "." ]]; then
return 0
else
- d=$(cd -P -- "$(dirname -- "${dir}")" >/dev/null && pwd -P)
- relative_dir "${d}" >/dev/null
- if [[ $? == 1 ]]; then
- yetus_debug "ERROR: ${dir} is not in ${BASEDIR}."
- return 1
- fi
dir=$(dirname "${dir}")
yetus_debug "Trying to skip: ${dir}"
fi
[06/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/d2114ff2
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/d2114ff2
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/d2114ff2
Branch: refs/heads/master
Commit: d2114ff2f0dedd5b58b6905bfc39d5ed56d734cb
Parents: 2cebff7 c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jul 31 14:44:17 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jul 31 14:44:17 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[18/50] [abbrv] yetus git commit: Merge branch 'trunk' into
HADOOP-12111
Posted by bu...@apache.org.
Merge branch 'trunk' into HADOOP-12111
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/ff2115cd
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/ff2115cd
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/ff2115cd
Branch: refs/heads/master
Commit: ff2115cde891a6cc3e5b1f0542ac3c435d633ed1
Parents: b7a0e94 c8215ad
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 10 09:43:33 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 10 09:43:33 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[20/50] [abbrv] yetus git commit: HADOOP-12310. final memory report
sometimes generates spurious errors (Kengo Seki via aw)
Posted by bu...@apache.org.
HADOOP-12310. final memory report sometimes generates spurious errors (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/b5713747
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/b5713747
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/b5713747
Branch: refs/heads/master
Commit: b57137478884281022f6a8c0944611fc630f1b46
Parents: b44bde5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Aug 10 09:46:06 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Aug 10 09:46:06 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/b5713747/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 1faf99f..446d5cf 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -395,9 +395,10 @@ function finish_footer_table
{
local maxmem
+ # `sort | head` can cause a broken pipe error, but we can ignore it just like compute_gitdiff.
# shellcheck disable=SC2016,SC2086
maxmem=$(find "${PATCH_DIR}" -type f -exec ${AWK} 'match($0, /^\[INFO\] Final Memory: [0-9]+/)
- { print substr($0, 22, RLENGTH-21) }' {} \; | sort -nr | head -n 1)
+ { print substr($0, 22, RLENGTH-21) }' {} \; | sort -nr 2>/dev/null | head -n 1)
if [[ -n ${maxmem} ]]; then
add_footer_table "Max memory used" "${maxmem}MB"