You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/06/27 09:09:55 UTC
[2/3] hadoop git commit: HADOOP-12113. update test-patch branch to
latest code (continued) (aw)
HADOOP-12113. update test-patch branch to latest code (continued) (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37578379
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37578379
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37578379
Branch: refs/heads/HADOOP-12111
Commit: 375783793a0b20049964c15eaa9150fbe299d7cf
Parents: 09a2e36
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jun 26 23:58:57 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jun 26 23:58:57 2015 -0700
----------------------------------------------------------------------
dev-support/docs/precommit-advanced.md | 164 ++++++++
dev-support/docs/precommit-architecture.md | 98 +++++
dev-support/docs/precommit-basic.md | 123 ++++++
dev-support/docs/precommit-glossary.md | 37 ++
dev-support/personality/flink.sh | 151 ++++++++
dev-support/personality/hadoop.sh | 217 +++++++++++
dev-support/personality/hbase.sh | 238 ++++++++++++
dev-support/personality/pig.sh | 57 +++
dev-support/personality/tajo.sh | 60 +++
dev-support/personality/tez.sh | 66 ++++
.../test-patch-docker/Dockerfile-endstub | 19 +
.../test-patch-docker/Dockerfile-startstub | 83 ++++
.../test-patch-docker/launch-test-patch.sh | 51 +++
.../test-patch-docker/test-patch-docker.sh | 383 +++++++++++++++++++
dev-support/test-patch.d/apache-rat.sh | 84 ++++
dev-support/test-patch.d/builtin-personality.sh | 157 ++++++++
dev-support/test-patch.d/findbugs.sh | 379 ++++++++++++++++++
dev-support/test-patch.d/github.sh | 112 ++++++
dev-support/test-patch.d/jira.sh | 190 +++++++++
dev-support/test-patch.d/xml.sh | 70 ++++
20 files changed, 2739 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/docs/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-advanced.md b/dev-support/docs/precommit-advanced.md
new file mode 100644
index 0000000..0a7eac5
--- /dev/null
+++ b/dev-support/docs/precommit-advanced.md
@@ -0,0 +1,164 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+test-patch
+==========
+
+* [Docker Support](#Docker_Support)
+* [Maven Profiles](#Maven_Profiles)
+* [Plug-ins](#Plug-ins)
+* [Configuring for Other Projects](#Configuring_for_Other_Projects)
+
+# Docker Support
+
+By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself into a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, the patch requires a newer version of Java.
+
+The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. Be aware that test-patch will copy this file and append its necessary hooks in order to execute.
+
+test-patch includes code to automatically manage broken/stale container images that are hanging around if it is run in --jenkins mode. In this way, if Docker fails to build the image, the disk space should eventually return.
+
+# Maven Profiles
+
+By default, test-patch will pass -Ptest-patch and -D${PROJECT_NAME}PatchProcess to Maven. This will allow you to configure special actions that should only happen when running underneath test-patch.
+
+
+# Plug-ins
+
+test-patch allows one to add to its basic feature set via plug-ins. There is a directory called test-patch.d off of the directory where test-patch.sh lives. Inside this directory one may place some bash shell fragments that, if setup with proper functions, will allow for test-patch to call it as necessary.
+
+
+Every plugin must have one line in order to be recognized:
+
+```bash
+add_plugin <pluginname>
+```
+
+This function call registers the `pluginname` so that test-patch knows that it exists. This plug-in name also acts as the key to the custom functions that you can define. For example:
+
+```bash
+function pluginname_filefilter
+```
+
+This function gets called for every file that a patch may contain. This allows the plug-in author to determine if this plug-in should be called, what files it might need to analyze, etc.
+
+Similarly, there are other functions that may be defined during the test-patch run:
+
+* pluginname_postcheckout
+ - executed prior to the patch being applied but after the git repository is setup. This is useful for any early error checking that might need to be done before any heavier work.
+
+* pluginname_preapply
+ - executed prior to the patch being applied. This is useful for any "before"-type data collection for later comparisons
+
+* pluginname_postapply
+ - executed after the patch has been applied. This is useful for any "after"-type data collection.
+
+* pluginname_postinstall
+ - executed after the mvn install test has been done. If any tests require the Maven repository to be up-to-date with the contents of the patch, this is the place.
+
+* pluginname_tests
+ - executed after the unit tests have completed.
+
+If the plug-in has some specific options, one can use following functions:
+
+* pluginname_usage
+
+ - executed when the help message is displayed. This is used to display the plug-in specific options for the user.
+
+* pluginname_parse_args
+
+ - executed prior to any other above functions except for pluginname_usage. This is useful for parsing the arguments passed from the user and setting up the execution environment.
+
+ HINT: It is recommend to make the pluginname relatively small, 10 characters at the most. Otherwise the ASCII output table may be skewed.
+
+
+# Configuring for Other Projects
+
+It is impossible for any general framework to be predictive about what types of special rules any given project may have, especially when it comes to ordering and Maven profiles. In order to assist non-Hadoop projects, a project `personality` should be added that enacts these custom rules.
+
+A personality consists of two functions. One that determines which test types to run and another that allows a project to dictate ordering rules, flags, and profiles on a per-module, per-test run.
+
+There can be only **one** of each personality function defined.
+
+## Test Determination
+
+The `personality_file_tests` function determines which tests to turn on based upon the file name. It is realtively simple. For example, to turn on a full suite of tests for Java files:
+
+```bash
+function personality_file_tests
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ add_test javac
+ add_test javadoc
+ add_test mvninstall
+ add_test unit
+ fi
+
+}
+```
+
+The `add_test` function is used to activate the standard tests. Additional plug-ins (such as checkstyle), will get queried on their own.
+
+## Module & Profile Determination
+
+Once the tests are determined, it is now time to pick which [modules](precommit-glossary.md#genericoutside-definitions) should get used. That's the job of the `personality_modules` function.
+
+```bash
+function personality_modules
+{
+
+ clear_personality_queue
+
+...
+
+ personality_enqueue_module <module> <flags>
+
+}
+```
+
+It takes exactly two parameters `repostatus` and `testtype`.
+
+The `repostatus` parameter tells the `personality` function exactly what state the repository is in. It can only be in one of two states: `branch` or `patch`. `branch` means the patch has not been applied. The `patch` state is after the patch has been applied.
+
+The `testtype` state tells the personality exactly which test is about to be executed.
+
+In order to communicate back to test-patch, there are two functions for the personality to use.
+
+The first is `clear_personality_queue`. This removes the previous test's configuration so that a new module queue may be built.
+
+The second is `personality_enqueue_module`. This function takes two parameters. The first parameter is the name of the module to add to this test's queue. The second parameter is an option list of additional flags to pass to Maven when processing it. `personality_enqueue_module` may be called as many times as necessary for your project.
+
+ NOTE: A module name of . signifies the root of the repository.
+
+For example, let's say your project uses a special configuration to skip unit tests (-DskipTests). Running unit tests during a javadoc build isn't very interesting. We can write a simple personality check to disable the unit tests:
+
+
+```bash
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+
+ if [[ ${testtype} == 'javadoc' ]]; then
+ personality_enqueue_module . -DskipTests
+ return
+ fi
+ ...
+
+```
+
+This function will tell test-patch that when the javadoc test is being run, do the documentation test at the base of the repository and make sure the -DskipTests flag is passed to Maven.
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/docs/precommit-architecture.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-architecture.md b/dev-support/docs/precommit-architecture.md
new file mode 100644
index 0000000..c134728
--- /dev/null
+++ b/dev-support/docs/precommit-architecture.md
@@ -0,0 +1,98 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Some Philosophy
+
+* Everyone's time is valuable. The quicker contributors can get feedback and iterate, the more likely their contribution will get checked in. A committer should be able to focus on the core issues of a contribution rather than details that might be able to be determined automatically.
+
+* Precommit checks should be fast. There is no value in testing parts of the source tree that are not immediately impacted by a change. Unit testing is the target. They are not a replacement for full builds, which is where integration tests should happen.
+
+* Many open source projects have a desire to have this capability. Why not generalize a solution?
+
+* In many build systems (especially with maven), a modular design has been picked. Why not leverage that design to make checks faster?
+
+* Projects that use the same language will, with a high degree of certainity, benefit from the same types of checks.
+
+* Portability matters.
+
+# Phases
+
+test-patch works effectively under several different phases:
+
+## Setup
+
+This is where test-patch configures and validates the environemnt. Some things done in this phase:
+
+* Defaults
+* Parameter handling
+* Importing plugins and personalities
+* Docker container launching
+* Re-exec support
+* Patch file downloading
+* git repository management (fresh pull, branch switching, etc)
+
+## Post-checkout
+
+Checks done here are *fatal*.
+
+This acts as a verification of all of the setup parts and is the final place to short-cut the full test cycle. The most significant built-in check done here is verifying the patch file is a valid.
+
+## Pre-apply
+
+This is where the 'before' work is handled. Some things done in this phase:
+
+* The first pass of files and modules that will get patched
+* Validation and information gathering of java, javadoc, site, the mvn repo, findbugs, etc.
+* Author checks
+* check for modified unit tests
+
+## Patch is Applied
+
+The patch gets applied. Then a second pass to determine which modules and files have been changed in order to handle any modules that might have added or moved.
+
+## Post-apply
+
+Now that the patch has been applied, many of the same checks performed in the Pre-apply step are done again to build an 'after' picture.
+
+* Validation and information gathering of java, javadoc, site, the mvn repo, findbugs, etc.
+
+## Post-install
+
+Some tests only work correctly when the repo is up-to-date. So
+mvn install is run to update the local repo and we enter this phase. Tests performed here:
+
+* Verification that maven eclipse integration still works
+* FindBugs
+
+## Unit Tests
+
+Since unit tests are generally the slowest part of the precommit process, they are run last. At this point, all the prerequisites to running them should be in place and ready to go.
+
+## Reporting
+
+Finally, the results are reported to the screen and, optionally, to JIRA.
+
+# Test Flow
+
+The basic workflow for many of the sub-items in individual phases are:
+
+1. print a header, so the end user knows that something is happening
+1. verify if the test is needed. If so, continue on. Otherwise, return success and let the next part of the phase execute.
+1. Ask the personality about what modules and what flags should get used
+1. Execute maven in the given modules with the given flags. Log the output and record the time and result code.
+1. Do any extra work as appropriate (diffs, counts, etc) and either accept the status and message given by the maven run or change the vote, message, log file, etc.
+1. Add the outcome(s) to the report generator
+
+As one can see, the modules list is one of the key inputs into what actually gets executed. As a result, projects must full flexibility in either adding, modifying, or even removing modules from the test list. If a personality removes the entire list of modules, then that test should just be ignored.
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/docs/precommit-basic.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-basic.md b/dev-support/docs/precommit-basic.md
new file mode 100644
index 0000000..ee2e063
--- /dev/null
+++ b/dev-support/docs/precommit-basic.md
@@ -0,0 +1,123 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+test-patch
+==========
+
+* [Purpose](#Purpose)
+* [Pre-requisites](#Pre-requisites)
+* [Basic Usage](#Basic_Usage)
+
+## Purpose
+
+As part of Hadoop's commit process, all patches to the source base go through a precommit test that does some (usually) light checking to make sure the proposed change does not break unit tests and/or passes some other prerequisites. This is meant as a preliminary check for committers so that the basic patch is in a known state. This check, called test-patch, may also be used by individual developers to verify a patch prior to sending to the Hadoop QA systems.
+
+Other projects have adopted a similar methodology after seeing great success in the Hadoop model. Some have even gone as far as forking Hadoop's precommit code and modifying it to meet their project's needs.
+
+This is a modification to Hadoop's version of test-patch so that we may bring together all of these forks under a common code base to help the community as a whole.
+
+
+## Pre-requisites
+
+test-patch has the following requirements:
+
+* Ant- or Maven-based project (and ant/maven installed)
+* git-based project (and git installed)
+* bash v3.2 or higher
+* findbugs 3.x installed
+* shellcheck installed
+* GNU diff
+* GNU patch
+* POSIX awk
+* POSIX grep
+* POSIX sed
+* wget
+* file command
+* smart-apply-patch.sh
+
+Maven plugins requirements:
+
+* Apache RAT
+* Apache FindBugs
+
+Optional:
+
+* Apache JIRA-based issue tracking
+* JIRA cli tools
+
+The locations of these files are (mostly) assumed to be in the file path, but may be overridden via command line options. For Solaris and Solaris-like operating systems, the default location for the POSIX binaries is in /usr/xpg4/bin.
+
+
+## Basic Usage
+
+This command will execute basic patch testing against a patch file stored in filename:
+
+```bash
+$ cd <your repo>
+$ dev-support/test-patch.sh --dirty-workspace --project=projectname <filename>
+```
+
+The `--dirty-workspace` flag tells test-patch that the repository is not clean and it is ok to continue. This version command does not run the unit tests.
+
+To do that, we need to provide the --run-tests command:
+
+
+```bash
+$ cd <your repo>
+$ dev-support/test-patch.sh --dirty-workspace --run-tests <filename>
+```
+
+This is the same command, but now runs the unit tests.
+
+A typical configuration is to have two repositories. One with the code you are working on and another, clean repository. This means you can:
+
+```bash
+$ cd <workrepo>
+$ git diff --no-prefix trunk > /tmp/patchfile
+$ cd ../<testrepo>
+$ <workrepo>/dev-support/test-patch.sh --basedir=<testrepo> --resetrepo /tmp/patchfile
+```
+
+We used two new options here. --basedir sets the location of the repository to use for testing. --resetrepo tells test patch that it can go into **destructive** mode. Destructive mode will wipe out any changes made to that repository, so use it with care!
+
+After the tests have run, there is a directory that contains all of the test-patch related artifacts. This is generally referred to as the patchprocess directory. By default, test-patch tries to make something off of /tmp to contain this content. Using the `--patchdir` command, one can specify exactly which directory to use. This is helpful for automated precommit testing so that the Jenkins or other automated workflow system knows where to look to gather up the output.
+
+## Providing Patch Files
+
+It is a fairly common practice within the Apache community to use Apache's JIRA instance to store potential patches. As a result, test-patch supports providing just a JIRA issue number. test-patch will find the *last* attachment, download it, then process it.
+
+For example:
+
+```bash
+$ test-patch.sh (other options) HADOOP-9905
+```
+
+... will process the patch file associated with this JIRA issue.
+
+
+A new practice is to use a service such as GitHub and its Pull Request (PR) feature. Luckily, test-patch supports URLs and many services like GitHub provide ways to provide unified diffs via URLs.
+
+For example:
+
+```bash
+$ test-patch.sh (other options) https://github.com/apache/flink/pull/773.patch
+```
+
+... will grab a unified diff of PR #773 and process it.
+
+## In Closing
+
+test-patch has many other features and command line options for the basic user. Many of these are self-explanatory. To see the list of options, run test-patch.sh without any options or with --help.
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/docs/precommit-glossary.md
----------------------------------------------------------------------
diff --git a/dev-support/docs/precommit-glossary.md b/dev-support/docs/precommit-glossary.md
new file mode 100644
index 0000000..ca3d15f
--- /dev/null
+++ b/dev-support/docs/precommit-glossary.md
@@ -0,0 +1,37 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Glossary
+
+## Generic/outside definitions:
+
+Apache's [new contributor documentation](https://community.apache.org/contributors/) and Maven's [glossary](https://maven.apache.org/glossary.html) are great places to start if you are new to Apache or Maven.
+
+* Module
+
+ Almost the same meaning as "sub-project" on Maven.
+
+## test-patch specific
+
+* Personality
+
+ A chunk of shell code that tells test-patch this particular project's needs and special requirements
+
+* Plug-ins
+
+ Shell code snippets that define new, not built-in test types.
+
+* Precommit
+
+ An automated process that verifies a patch is "good" prior to a committer looking at it.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/personality/flink.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/flink.sh b/dev-support/personality/flink.sh
new file mode 100644
index 0000000..a32e2d6
--- /dev/null
+++ b/dev-support/personality/flink.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=master
+#shellcheck disable=SC2034
+ISSUE_RE='^FLINK-[0-9]+$'
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE=""
+
+add_plugin flinklib
+
+function fliblib_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$
+ || ${filename} =~ \.scala$
+ || ${filename} =~ pom.xml$ ]]; then
+ add_test flinklib
+ fi
+}
+
+function flinklib_count
+{
+ find "${BASEDIR}" \
+ | ${GREP} "/lib/" \
+ | ${GREP} -v "_qa_workdir" \
+ | wc -l
+}
+
+function flinklib_preapply
+{
+ start_clock
+ big_console_header "${PATCH_BRANCH} flink library dependencies"
+
+ verify_needed_test flinklib
+ if [[ $? == 0 ]]; then
+ echo "Patch does not need flinklib testing."
+ return 0
+ fi
+
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/branch-flinklib-root.txt" \
+ "${MVN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
+ if [[ $? != 0 ]]; then
+ add_vote_table -1 flinklib "Unable to determine flink libs in ${PATCH_BRANCH}."
+ fi
+ FLINK_PRE_LIB_FILES=$(flinklib_count)
+ popd >/dev/null
+}
+
+function flinklib_postapply
+{
+ start_clock
+ big_console_header "Patch flink library dependencies"
+
+ verify_needed_test flinklib
+ if [[ $? == 0 ]]; then
+ echo "Patch does not need flinklib testing."
+ return 0
+ fi
+
+ pushd "${BASEDIR}" >/dev/null
+ echo_and_redirect "${PATCH_DIR}/patch-flinklib-root.txt" \
+ "${MVN}" "${MAVEN_ARGS[@]}" package -DskipTests -Dmaven.javadoc.skip=true -Ptest-patch
+ FLINK_POST_LIB_FILES=$(flinklib_count)
+ popd >/dev/null
+
+
+ if [[ "${FLINK_POST_LIB_FILES}" -gt "${FLINK_PRE_LIB_FILES}" ]]; then
+ add_vote_table -1 flinklib "Patch increases lib folder dependencies from " \
+ "${FLINK_PRE_LIB_FILES} to ${FLINK_POST_LIB_FILES}"
+ return 1
+ elif [[ "${FLINK_POST_LIB_FILES}" -eq "${FLINK_PRE_LIB_FILES}" ]]; then
+ add_vote_table 0 flinklib "Patch did not change lib dependencies" \
+ " (still ${FLINK_PRE_LIB_FILES})"
+ else
+ add_vote_table +1 flinklib "Patch decreases lib folder dependencies by " \
+ "$((FLINK_PRE_LIB_FILES-FLINK_POST_LIB_FILES))."
+ fi
+ return 0
+}
+
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ mvninstall)
+ if [[ ${repostatus} == branch ]]; then
+ personality_enqueue_module . -DskipTests
+ return
+ fi
+ return
+ ;;
+ releaseaudit)
+ # this is very fast and provides the full path if we do it from
+ # the root of the source
+ personality_enqueue_module .
+ return
+ ;;
+ unit)
+ if [[ ${TEST_PARALLEL} == "true" ]] ; then
+ extra="-Pparallel-tests"
+ if [[ -n ${TEST_THREADS:-} ]]; then
+ extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
+ fi
+ fi
+ ;;
+ *)
+ extra="-DskipTests"
+ ;;
+ esac
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
+
+function personality_file_tests
+{
+ local filename=$1
+
+ yetus_debug "Using personality_file_tests, but calling the built-in:"
+ builtin_personality_file_tests "${1}"
+
+ if [[ ${filename} =~ \.scala$ ]]; then
+ add_test unit
+ fi
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hadoop.sh b/dev-support/personality/hadoop.sh
new file mode 100644
index 0000000..059d051
--- /dev/null
+++ b/dev-support/personality/hadoop.sh
@@ -0,0 +1,217 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Override these to match Apache Hadoop's requirements
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=trunk
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE="https://wiki.apache.org/hadoop/HowToContribute"
+#shellcheck disable=SC2034
+ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$'
+
+HADOOP_MODULES=""
+
+function hadoop_module_manipulation
+{
+ local need_common=0
+ local module
+ local hdfs_modules
+ local ordered_modules
+ local tools_modules
+ local passed_modules=${CHANGED_MODULES}
+
+ yetus_debug "hmm: starting list: ${passed_modules}"
+
+ # if one of our modules is ., then shortcut:
+ # ignore the rest and just set it to everything.
+ if [[ ${CHANGED_MODULES} == ' . ' ]]; then
+ HADOOP_MODULES='.'
+ return
+ fi
+
+ # ${CHANGED_MODULES} is already sorted and uniq'd.
+ # let's remove child modules if we're going to
+ # touch their parent
+ for module in ${CHANGED_MODULES}; do
+ yetus_debug "Stripping ${module}"
+ # shellcheck disable=SC2086
+ passed_modules=$(echo ${passed_modules} | tr ' ' '\n' | ${GREP} -v ${module}/ )
+ done
+
+ for module in ${passed_modules}; do
+ yetus_debug "Personality ordering ${module}"
+ if [[ ${module} == hadoop-hdfs-project* ]]; then
+ hdfs_modules="${hdfs_modules} ${module}"
+ need_common=1
+ elif [[ ${module} == hadoop-common-project/hadoop-common
+ || ${module} == hadoop-common-project ]]; then
+ ordered_modules="${ordered_modules} ${module}"
+ building_common=1
+ elif [[ ${module} == hadoop-tools* ]]; then
+ tools_modules="${tools_modules} ${module}"
+ else
+ ordered_modules="${ordered_modules} ${module}"
+ fi
+ done
+
+ ordered_modules="${ordered_modules} ${hdfs_modules} ${tools_modules}"
+
+ if [[ ${need_common} -eq 1
+ && ${building_common} -eq 0 ]]; then
+ ordered_modules="hadoop-common-project/hadoop-common ${ordered_modules}"
+ fi
+
+ yetus_debug "hmm: ${ordered_modules}"
+ HADOOP_MODULES=${ordered_modules}
+}
+
+function hadoop_javac_ordering
+{
+ local special=$1
+ local ordered_modules
+ local module
+
+ # Based upon HADOOP-11937
+ #
+ # Some notes:
+ #
+ # - getting fuse to compile on anything but Linux
+ # is always tricky.
+ # - Darwin assumes homebrew is in use.
+ # - HADOOP-12027 required for bzip2 on OS X.
+ # - bzip2 is broken in lots of places.
+ # e.g, HADOOP-12027 for OS X. so no -Drequire.bzip2
+ #
+
+ for module in ${HADOOP_MODULES}; do
+ if [[ ${JENKINS} == true
+ && ${DOCKERSUPPORT} == false ]]; then
+ # shellcheck disable=SC2086
+ personality_enqueue_module "${module}" ${special} \
+ -Pnative \
+ -Drequire.snappy -Drequire.openssl -Drequire.fuse \
+ -Drequire.test.libhadoop
+ else
+ case ${OSTYPE} in
+ Linux)
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${special} \
+ -Pnative -Drequire.libwebhdfs \
+ -Drequire.snappy -Drequire.openssl -Drequire.fuse \
+ -Drequire.test.libhadoop
+ ;;
+ Darwin)
+ JANSSON_INCLUDE_DIR=/usr/local/opt/jansson/include
+ JANSSON_LIBRARY=/usr/local/opt/jansson/lib
+ export JANSSON_LIBRARY JANSSON_INCLUDE_DIR
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${special} \
+ -Pnative -Drequire.snappy \
+ -Drequire.openssl \
+ -Dopenssl.prefix=/usr/local/opt/openssl/ \
+ -Dopenssl.include=/usr/local/opt/openssl/include \
+ -Dopenssl.lib=/usr/local/opt/openssl/lib \
+ -Drequire.libwebhdfs -Drequire.test.libhadoop
+ ;;
+ *)
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${special} \
+ -Pnative \
+ -Drequire.snappy -Drequire.openssl \
+ -Drequire.libwebhdfs -Drequire.test.libhadoop
+ ;;
+ esac
+ fi
+ done
+}
+
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+ local fn
+ local i
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ javac)
+ if [[ ${BUILD_NATIVE} == true ]]; then
+ hadoop_module_manipulation
+ hadoop_javac_ordering -DskipTests
+ return
+ fi
+ extra="-DskipTests"
+ ;;
+ javadoc)
+ if [[ ${repostatus} == patch ]]; then
+ echo "javadoc pre-reqs:"
+ for i in hadoop-project \
+ hadoop-common-project/hadoop-annotations; do
+ fn=$(module_file_fragment "${i}")
+ pushd "${BASEDIR}/${i}" >/dev/null
+ echo "cd ${i}"
+ echo_and_redirect "${PATCH_DIR}/maven-${fn}-install.txt" \
+ "${MVN}" "${MAVEN_ARGS[@]}" install
+ popd >/dev/null
+ done
+ fi
+ extra="-Pdocs -DskipTests"
+ ;;
+ mvninstall)
+ extra="-DskipTests"
+ if [[ ${repostatus} == branch ]]; then
+ HADOOP_MODULES=.
+ hadoop_javac_ordering -DskipTests
+ return
+ fi
+ ;;
+ releaseaudit)
+ # this is very fast and provides the full path if we do it from
+ # the root of the source
+ personality_enqueue_module .
+ return
+ ;;
+ unit)
+ if [[ ${TEST_PARALLEL} == "true" ]] ; then
+ extra="-Pparallel-tests"
+ if [[ -n ${TEST_THREADS:-} ]]; then
+ extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
+ fi
+ fi
+ if [[ ${BUILD_NATIVE} == true ]]; then
+ hadoop_module_manipulation
+ # shellcheck disable=SC2086
+ hadoop_javac_ordering ${extra}
+ return
+ fi
+ ;;
+ *)
+ extra="-DskipTests"
+ ;;
+ esac
+
+ hadoop_module_manipulation
+ for module in ${HADOOP_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/personality/hbase.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/hbase.sh b/dev-support/personality/hbase.sh
new file mode 100644
index 0000000..46ad390
--- /dev/null
+++ b/dev-support/personality/hbase.sh
@@ -0,0 +1,238 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=master
+#shellcheck disable=SC2034
+ISSUE_RE='^HBASE-[0-9]+$'
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE=""
+
+# All supported Hadoop versions that we want to test the compilation with
+HADOOP2_VERSIONS="2.4.1 2.5.2 2.6.0"
+
+# Override the maven options
+MAVEN_OPTS="${MAVEN_OPTS:-"-Xmx3100M"}"
+
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ javac)
+ personality_enqueue_module . -DskipTests
+ return
+ ;;
+ mvninstall)
+ extra="-DskipTests -DHBasePatchProcess"
+ if [[ ${repostatus} == branch ]]; then
+ personality_enqueue_module . "${extra}"
+ return
+ fi
+ return
+ ;;
+ releaseaudit)
+ # this is very fast and provides the full path if we do it from
+ # the root of the source
+ personality_enqueue_module . -DHBasePatchProcess
+ return
+ ;;
+ unit)
+ if [[ ${TEST_PARALLEL} == "true" ]] ; then
+ extra="-Pparallel-tests -DHBasePatchProcess"
+ if [[ -n ${TEST_THREADS:-} ]]; then
+ extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
+ fi
+ fi
+ ;;
+ *)
+ extra="-DskipTests -DHBasePatchProcess"
+ ;;
+ esac
+
+ for module in ${CHANGED_MODULES}; do
+
+ # skip findbugs on hbase-shell
+ if [[ ${module} == hbase-shell
+ && ${testtype} == findbugs ]]; then
+ continue
+ else
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ fi
+ done
+}
+
+###################################################
+
+add_plugin hadoopcheck
+
+function hadoopcheck_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test hadoopcheck
+ fi
+}
+
+function hadoopcheck_postapply
+{
+ local HADOOP2_VERSION
+ local logfile
+ local count
+ local result=0
+
+ big_console_header "Compiling against various Hadoop versions"
+
+ export MAVEN_OPTS="${MAVEN_OPTS}"
+ for HADOOP2_VERSION in ${HADOOP2_VERSIONS}; do
+ logfile="${PATCH_DIR}/patch-javac-${HADOOP2_VERSION}.txt"
+ echo_and_redirect "${logfile}" \
+ "${MVN}" clean install \
+ -DskipTests -DHBasePatchProcess \
+ -Dhadoop-two.version="${HADOOP2_VERSION}"
+ count=$(${GREP} -c ERROR "${logfile}")
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table -1 hadoopcheck "Patch causes ${count} errors with Hadoop v${HADOOP2_VERSION}."
+ ((result=result+1))
+ fi
+ done
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ add_vote_table +1 hadoopcheck "Patch does not cause any errors with Hadoop ${HADOOP2_VERSIONS}."
+ return 0
+}
+
+######################################
+
+add_plugin hbaseprotoc
+
+function hbaseprotoc_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.proto$ ]]; then
+ add_test hbaseprotoc
+ fi
+}
+
+function hbaseprotoc_postapply
+{
+ local i=0
+ local fn
+ local module
+ local logfile
+ local count
+ local results
+
+ big_console_header "Patch HBase protoc plugin"
+
+ start_clock
+
+ verify_needed_test hbaseprotoc
+ if [[ $? == 0 ]]; then
+ echo "Patch does not need hbaseprotoc testing."
+ return 0
+ fi
+
+ personality_modules patch hbaseprotoc
+ modules_workers patch hbaseprotoc -DskipTests -Pcompile-protobuf -X -DHBasePatchProcess
+
+ # shellcheck disable=SC2153
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+ module=${MODULE[$i]}
+ fn=$(module_file_fragment "${module}")
+ logfile="${PATCH_DIR}/patch-hbaseprotoc-${fn}.txt"
+
+ count=$(${GREP} -c ERROR "${logfile}")
+
+ if [[ ${count} -gt 0 ]]; then
+ module_status ${i} -1 "patch-hbaseprotoc-${fn}.txt" "Patch generated "\
+ "${count} new protoc errors in ${module}."
+ ((results=results+1))
+ fi
+ ((i=i+1))
+ done
+
+ modules_messages patch hbaseprotoc true
+ if [[ ${results} -gt 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+######################################
+
+add_plugin hbaseanti
+
+function hbaseanti_filefilter
+{
+ local filename=$1
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test hbaseanti
+ fi
+}
+
+function hbaseanti_preapply
+{
+ local warnings
+ local result
+
+ big_console_header "Checking for known anti-patterns"
+
+ start_clock
+
+ verify_needed_test hbaseanti
+ if [[ $? == 0 ]]; then
+ echo "Patch does not need hbaseanti testing."
+ return 0
+ fi
+
+ warnings=$(${GREP} 'new TreeMap<byte.*()' "${PATCH_DIR}/patch")
+ if [[ ${warnings} -gt 0 ]]; then
+ add_vote_table -1 hbaseanti "" "The patch appears to have anti-pattern where BYTES_COMPARATOR was omitted: ${warnings}."
+ ((result=result+1))
+ fi
+
+ warnings=$(${GREP} 'import org.apache.hadoop.classification' "${PATCH_DIR}/patch")
+ if [[ ${warnings} -gt 0 ]]; then
+ add_vote_table -1 hbaseanti "" "The patch appears use Hadoop classification instead of HBase: ${warnings}."
+ ((result=result+1))
+ fi
+
+ if [[ ${result} -gt 0 ]]; then
+ return 1
+ fi
+
+ add_vote_table +1 hbaseanti "" "Patch does not have any anti-patterns."
+ return 0
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/personality/pig.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/pig.sh b/dev-support/personality/pig.sh
new file mode 100644
index 0000000..69dabf1
--- /dev/null
+++ b/dev-support/personality/pig.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=trunk
+#shellcheck disable=SC2034
+ISSUE_RE='^PIG-[0-9]+$'
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE=""
+#shellcheck disable=SC2034
+BUILDTOOL=ant
+
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ extra="-DPigPatchProcess= "
+
+ case ${testtype} in
+ findbugs)
+ # shellcheck disable=SC2034
+ ANT_FINDBUGSXML="${BASEDIR}/build/test/findbugs/pig-findbugs-report.xml"
+ extra="-Dfindbugs.home=${FINDBUGS_HOME}"
+ ;;
+ javac)
+ extra="${extra} -Djavac.args=-Xlint -Dcompile.c++=yes clean tar"
+ ;;
+ javadoc)
+ extra="${extra} -Dforrest.home=${FORREST_HOME}"
+ ;;
+ unit)
+ extra="${extra} -Dtest.junit.output.format=xml -Dcompile.c++=yes -Dtest.output=yes test-core"
+ ;;
+ esac
+
+ # shellcheck disable=SC2086
+ personality_enqueue_module . ${extra}
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/personality/tajo.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/tajo.sh b/dev-support/personality/tajo.sh
new file mode 100644
index 0000000..719bada
--- /dev/null
+++ b/dev-support/personality/tajo.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=master
+#shellcheck disable=SC2034
+ISSUE_RE='^TAJO-[0-9]+$'
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TAJO/How+to+Contribute+to+Tajo"
+
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ mvninstall)
+ extra="-DskipTests"
+ if [[ ${repostatus} == branch ]]; then
+ personality_enqueue_module . "${extra}"
+ return
+ fi
+ return
+ ;;
+ releaseaudit)
+ # this is very fast and provides the full path if we do it from
+ # the root of the source
+ personality_enqueue_module .
+ return
+ ;;
+ unit)
+ ;;
+ *)
+ extra="-DskipTests"
+ ;;
+ esac
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/personality/tez.sh
----------------------------------------------------------------------
diff --git a/dev-support/personality/tez.sh b/dev-support/personality/tez.sh
new file mode 100644
index 0000000..77ad624
--- /dev/null
+++ b/dev-support/personality/tez.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#shellcheck disable=SC2034
+PATCH_BRANCH_DEFAULT=master
+#shellcheck disable=SC2034
+ISSUE_RE='^TEZ-[0-9]+$'
+#shellcheck disable=SC2034
+HOW_TO_CONTRIBUTE="https://cwiki.apache.org/confluence/display/TEZ/How+to+Contribute+to+Tez"
+
+function personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+ local extra=""
+
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ case ${testtype} in
+ mvninstall)
+ extra="-DskipTests"
+ if [[ ${repostatus} == branch ]]; then
+ personality_enqueue_module . "${extra}"
+ return
+ fi
+ return
+ ;;
+ releaseaudit)
+ # this is very fast and provides the full path if we do it from
+ # the root of the source
+ personality_enqueue_module .
+ return
+ ;;
+ unit)
+ if [[ ${TEST_PARALLEL} == "true" ]] ; then
+ extra="-Pparallel-tests"
+ if [[ -n ${TEST_THREADS:-} ]]; then
+ extra="${extra} -DtestsThreadCount=${TEST_THREADS}"
+ fi
+ fi
+ ;;
+ *)
+ extra="-DskipTests"
+ ;;
+ esac
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module} ${extra}
+ done
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch-docker/Dockerfile-endstub
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/Dockerfile-endstub b/dev-support/test-patch-docker/Dockerfile-endstub
new file mode 100644
index 0000000..aa0463e
--- /dev/null
+++ b/dev-support/test-patch-docker/Dockerfile-endstub
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ADD launch-test-patch.sh /testptch/launch-test-patch.sh
+RUN chmod a+rx /testptch/launch-test-patch.sh
+CMD /testptch/launch-test-patch.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/Dockerfile-startstub b/dev-support/test-patch-docker/Dockerfile-startstub
new file mode 100644
index 0000000..1afd435
--- /dev/null
+++ b/dev-support/test-patch-docker/Dockerfile-startstub
@@ -0,0 +1,83 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:trusty
+
+WORKDIR /root
+
+######
+# Install common dependencies from packages
+######
+RUN apt-get update && apt-get install --no-install-recommends -y \
+ git curl ant make maven \
+ cmake gcc g++ protobuf-compiler \
+ build-essential libtool \
+ zlib1g-dev pkg-config libssl-dev \
+ snappy libsnappy-dev \
+ bzip2 libbz2-dev \
+ libjansson-dev \
+ fuse libfuse-dev \
+ libcurl4-openssl-dev \
+ python python2.7
+
+# Fixing the Apache commons / Maven dependency problem under Ubuntu:
+# See http://wiki.apache.org/commons/VfsProblems
+RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
+
+#######
+# Oracle Java
+#######
+
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository -y ppa:webupd8team/java
+RUN apt-get update
+
+
+# Auto-accept the Oracle JDK license
+RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
+RUN apt-get install -y oracle-java7-installer
+
+# Auto-accept the Oracle JDK license
+RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
+RUN apt-get install -y oracle-java8-installer
+
+######
+# Install findbugs
+######
+RUN mkdir -p /opt/findbugs && \
+ wget https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
+ -O /opt/findbugs.tar.gz && \
+ tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
+ENV FINDBUGS_HOME /opt/findbugs
+
+####
+# Install shellcheck
+####
+RUN apt-get install -y cabal-install
+RUN cabal update && cabal install shellcheck --global
+
+#####
+# Install JIRA CLI
+#####
+
+RUN mkdir -p /opt/jiracli && \
+ wget https://bobswift.atlassian.net/wiki/download/attachments/16285777/jira-cli-2.2.0-distribution.zip \
+ -O /tmp/jiracli.zip && \
+ unzip -qq -d /opt/jiracli /tmp/jiracli.zip && \
+ ln -s /opt/jiracli/jira-cli-2.2.0 /opt/jiracli/latest && \
+ chmod -R a+rx /opt/jiracli/jira-cli-2.2.0
+ENV JIRACLI_HOME /opt/jiracli/latest
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch-docker/launch-test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/launch-test-patch.sh b/dev-support/test-patch-docker/launch-test-patch.sh
new file mode 100644
index 0000000..410149b
--- /dev/null
+++ b/dev-support/test-patch-docker/launch-test-patch.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cd "${BASEDIR}"
+
+if [[ -n ${JAVA_HOME}
+ && ! -d ${JAVA_HOME} ]]; then
+ echo "JAVA_HOME: ${JAVA_HOME} does not exist. Dockermode: attempting to switch to another." 1>&2
+ JAVA_HOME=""
+fi
+
+if [[ -z ${JAVA_HOME} ]]; then
+ JAVA_HOME=$(find /usr/lib/jvm/ -name "java-*" -type d | tail -1)
+ export JAVA_HOME
+fi
+
+# Avoid out of memory errors in builds
+MAVEN_OPTS=${MAVEN_OPTS:-"-Xms256m -Xmx1g"}
+export MAVEN_OPTS
+
+# strip out --docker param to prevent re-exec again
+TESTPATCHMODE=${TESTPATCHMODE/--docker }
+
+
+cd "${BASEDIR}"
+PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
+
+cd "${PATCH_DIR}/precommit/"
+#shellcheck disable=SC2086
+"${PATCH_DIR}/precommit/test-patch.sh" \
+ --reexec \
+ --dockermode ${TESTPATCHMODE} \
+ --basedir="${BASEDIR}" \
+ --patch-dir="${PATCH_DIR}" \
+ --java-home="${JAVA_HOME}" \
+ --plugins="${PATCH_DIR}/precommit/user-plugins" \
+ --jira-cmd=/opt/jiracli/jira-cli-2.2.0/jira.sh
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch-docker/test-patch-docker.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch-docker/test-patch-docker.sh b/dev-support/test-patch-docker/test-patch-docker.sh
new file mode 100644
index 0000000..f8e46b4
--- /dev/null
+++ b/dev-support/test-patch-docker/test-patch-docker.sh
@@ -0,0 +1,383 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DID=${RANDOM}
+
+## @description Print a message to stderr if --debug is turned on
+## @audience private
+## @stability stable
+## @replaceable no
+## @param string
+function yetus_debug
+{
+ if [[ -n "${TP_SHELL_SCRIPT_DEBUG}" ]]; then
+ echo "[$(date) DEBUG]: $*" 1>&2
+ fi
+}
+
+## @description Run docker with some arguments, and
+## @description optionally send to debug
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function dockercmd
+{
+ yetus_debug "docker $*"
+ docker "$@"
+}
+
+## @description Handle command line arguments
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --debug)
+ TP_SHELL_SCRIPT_DEBUG=true
+ ;;
+ --dockerversion=*)
+ DOCKER_VERSION=${i#*=}
+ ;;
+ --help|-help|-h|help|--h|--\?|-\?|\?)
+ yetus_usage
+ exit 0
+ ;;
+ --java-home=*)
+ JAVA_HOME=${i#*=}
+ ;;
+ --patch-dir=*)
+ PATCH_DIR=${i#*=}
+ ;;
+ --project=*)
+ PROJECT_NAME=${i#*=}
+ ;;
+ *)
+ ;;
+ esac
+ done
+}
+
+## @description Stop and delete all defunct containers
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function stop_exited_containers
+{
+ local line
+ local id
+ local value
+ local size
+
+ echo "Docker containers in exit state:"
+
+ dockercmd ps -a | grep Exited
+
+ # stop *all* containers that are in exit state for
+ # more than > 8 hours
+ while read line; do
+ id=$(echo "${line}" | cut -f1 -d' ')
+ value=$(echo "${line}" | cut -f2 -d' ')
+ size=$(echo "${line}" | cut -f3 -d' ')
+
+ if [[ ${size} =~ day
+ || ${size} =~ week
+ || ${size} =~ month
+ || ${size} =~ year ]]; then
+ echo "Removing docker ${id}"
+ dockercmd rm "${id}"
+ fi
+
+ if [[ ${size} =~ hours
+ && ${value} -gt 8 ]]; then
+ echo "Removing docker ${id}"
+ dockercmd rm "${id}"
+ fi
+ done < <(
+ dockercmd ps -a \
+ | grep Exited \
+ | sed -e 's,ago,,g' \
+ | awk '{print $1" "$(NF - 2)" "$(NF - 1)}')
+}
+
+## @description Remove all containers that are not
+## @description are not running + older than 1 day
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function rm_old_containers
+{
+ local line
+ local id
+ local value
+ local size
+
+ while read line; do
+ id=$(echo "${line}" | cut -f1 -d, )
+ state=$(echo "${line}" | cut -f2 -d, )
+ stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. )
+
+ # believe it or not, date is not even close to standardized...
+ if [[ $(uname -s) == Linux ]]; then
+
+ # GNU date
+ stoptime=$(date -d "${stoptime}" "+%s")
+ else
+
+ # BSD date
+ stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s")
+ fi
+
+ if [[ ${state} == false ]]; then
+ curtime=$(date "+%s")
+ ((difftime = curtime - stoptime))
+ if [[ ${difftime} -gt 86400 ]]; then
+ echo "Removing docker ${id}"
+ dockercmd rm "${id}"
+ fi
+ fi
+ done < <(
+ # see https://github.com/koalaman/shellcheck/issues/375
+ # shellcheck disable=SC2046
+ dockercmd inspect \
+ -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \
+ $(dockercmd ps -qa) 2>/dev/null)
+}
+
+## @description Remove untagged/unused images
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function remove_untagged_images
+{
+ # this way is a bit more compatible with older docker versions
+ dockercmd images | tail -n +2 | awk '$1 == "<none>" {print $3}' | \
+ xargs --no-run-if-empty docker rmi
+}
+
+## @description Remove defunct tagged images
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function remove_old_tagged_images
+{
+ local line
+ local id
+ local created
+
+ while read line; do
+ id=$(echo "${line}" | awk '{print $1}')
+ created=$(echo "${line}" | awk '{print $5}')
+
+ if [[ ${created} =~ week
+ || ${created} =~ month
+ || ${created} =~ year ]]; then
+ echo "Removing docker image ${id}"
+ dockercmd rmi "${id}"
+ fi
+
+ if [[ ${id} =~ test-patch-base-${PROJECT_NAME}-date ]]; then
+ if [[ ${created} =~ day
+ || ${created} =~ hours ]]; then
+ echo "Removing docker image ${id}"
+ dockercmd rmi "${id}"
+ fi
+ fi
+ done < <(dockercmd images)
+
+}
+
+## @description Performance docker maintenance on Jenkins
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function cleanup_apache_jenkins_docker
+{
+ echo "=========================="
+ echo "Docker Images:"
+ dockercmd images
+ echo "=========================="
+ echo "Docker Containers:"
+ dockercmd ps -a
+ echo "=========================="
+
+ stop_exited_containers
+
+ rm_old_containers
+
+ remove_untagged_images
+
+ remove_old_tagged_images
+}
+
+## @description Clean up our old images used for patch testing
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function cleanup_test_patch_images
+{
+ local images
+ local imagecount
+ local rmimage
+ local rmi
+
+ # we always want to leave at least one of our images
+ # so that the whole thing doesn't have to be rebuilt.
+ # This also let's us purge any old images so that
+ # we can get fresh stuff sometimes
+ images=$(dockercmd images | grep --color=none "test-patch-tp-${PROJECT_NAME}" | awk '{print $1}') 2>&1
+
+ # shellcheck disable=SC2086
+ imagecount=$(echo ${images} | tr ' ' '\n' | wc -l)
+ ((imagecount = imagecount - 1 ))
+
+ # shellcheck disable=SC2086
+ rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount})
+ for rmi in ${rmimage}
+ do
+ echo "Removing image ${rmi}"
+ dockercmd rmi "${rmi}"
+ done
+}
+
+## @description Perform pre-run maintenance to free up
+## @description resources. With --jenkins, it is a lot
+## @description more destructive.
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function cleanup
+{
+ if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
+ cleanup_apache_jenkins_docker
+ fi
+
+ cleanup_test_patch_images
+}
+
+## @description Deterine the user name and user id of the user
+## @description that the docker container should use
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function determine_user
+{
+ # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some
+ # ... person ... sets it to an account that doesn't actually exist.
+ # so instead, we need to try and override it with something that's
+ # probably close to reality.
+ if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
+ USER=$(id | cut -f2 -d\( | cut -f1 -d\))
+ fi
+
+ if [[ "$(uname -s)" == "Linux" ]]; then
+ USER_NAME=${SUDO_USER:=$USER}
+ USER_ID=$(id -u "${USER_NAME}")
+ GROUP_ID=$(id -g "${USER_NAME}")
+ else # boot2docker uid and gid
+ USER_NAME=${USER}
+ USER_ID=1000
+ GROUP_ID=50
+ fi
+}
+
+## @description Determine the revision of a dockerfile
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function getdockerfilerev
+{
+ grep 'TEST_PATCH_PRIVATE: gitrev=' \
+ "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \
+ | cut -f2 -d=
+}
+
+## @description Start a test patch docker container
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function run_image
+{
+ local dockerfilerev
+ local baseimagename
+
+ dockerfilerev=$(getdockerfilerev)
+
+ baseimagename="test-patch-base-${PROJECT_NAME}-${dockerfilerev}"
+
+ # make a base image, if it isn't available
+ dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker"
+
+ # using the base image, make one that is patch specific
+ dockercmd build -t "test-patch-tp-${PROJECT_NAME}-${DID}" - <<PatchSpecificDocker
+FROM ${baseimagename}
+RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
+RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME}
+RUN chown -R ${USER_NAME} /home/${USER_NAME}
+ENV HOME /home/${USER_NAME}
+USER ${USER_NAME}
+PatchSpecificDocker
+
+ if [[ ${PATCH_DIR} =~ ^/ ]]; then
+ dockercmd run --rm=true -i \
+ -v "${PWD}:/testptch/${PROJECT_NAME}" \
+ -v "${PATCH_DIR}:/testptch/patchprocess" \
+ -v "${HOME}/.m2:${HOME}/.m2" \
+ -u "${USER_NAME}" \
+ -w "/testptch/${PROJECT_NAME}" \
+ --env=BASEDIR="/testptch/${PROJECT_NAME}" \
+ --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
+ --env=JAVA_HOME="${JAVA_HOME}" \
+ --env=PATCH_DIR=/testptch/patchprocess \
+ --env=PROJECT_NAME="${PROJECT_NAME}" \
+ --env=TESTPATCHMODE="${TESTPATCHMODE}" \
+ "test-patch-tp-${PROJECT_NAME}-${DID}"
+ else
+ dockercmd run --rm=true -i \
+ -v "${PWD}:/testptch/${PROJECT_NAME}" \
+ -v "${HOME}/.m2:${HOME}/.m2" \
+ -u "${USER_NAME}" \
+ -w "/testptch/${PROJECT_NAME}" \
+ --env=BASEDIR="/testptch/${PROJECT_NAME}" \
+ --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
+ --env=JAVA_HOME="${JAVA_HOME}" \
+ --env=PATCH_DIR="${PATCH_DIR}" \
+ --env=PROJECT_NAME="${PROJECT_NAME}" \
+ --env=TESTPATCHMODE="${TESTPATCHMODE}" \
+ "test-patch-tp-${PROJECT_NAME}-${DID}"
+ fi
+}
+
+parse_args "$@"
+cleanup
+determine_user
+run_image
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch.d/apache-rat.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/apache-rat.sh b/dev-support/test-patch.d/apache-rat.sh
new file mode 100644
index 0000000..27349bf
--- /dev/null
+++ b/dev-support/test-patch.d/apache-rat.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ ${BUILDTOOL} == maven
+ || ${BUILDTOOL} == ant ]]; then
+ add_plugin asflicense
+ add_test asflicense
+fi
+
+## @description Verify all files have an Apache License
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function asflicense_postapply
+{
+ local numpatch
+
+ big_console_header "Determining number of patched ASF License errors"
+
+ start_clock
+
+ personality_modules patch asflicense
+ case ${BUILDTOOL} in
+ maven)
+ modules_workers patch asflicense apache-rat:check
+ ;;
+ ant)
+ modules_workers patch asflicense releaseaudit
+ ;;
+ *)
+ return 0
+ ;;
+ esac
+
+ # RAT fails the build if there are license problems.
+ # so let's take advantage of that a bit.
+ if [[ $? == 0 ]]; then
+ add_vote_table 1 asflicense "Patch does not generate ASF License warnings."
+ return 0
+ fi
+
+ #shellcheck disable=SC2038
+ find "${BASEDIR}" -name rat.txt -o -name releaseaudit_report.txt \
+ | xargs cat > "${PATCH_DIR}/patch-asflicense.txt"
+
+ if [[ -s "${PATCH_DIR}/patch-asflicense.txt" ]] ; then
+ numpatch=$("${GREP}" -c '\!?????' "${PATCH_DIR}/patch-asflicense.txt")
+ echo ""
+ echo ""
+ echo "There appear to be ${numpatch} ASF License warnings after applying the patch."
+ if [[ -n ${numpatch}
+ && ${numpatch} -gt 0 ]] ; then
+ add_vote_table -1 asflicense "Patch generated ${numpatch} ASF License warnings."
+
+ echo "Lines that start with ????? in the ASF License "\
+ "report indicate files that do not have an Apache license header:" \
+ > "${PATCH_DIR}/patch-asflicense-problems.txt"
+
+ ${GREP} '\!?????' "${PATCH_DIR}/patch-asflicense.txt" \
+ >> "${PATCH_DIR}/patch-asflicense-problems.txt"
+
+ add_footer_table asflicense "@@BASE@@/patch-asflicense-problems.txt"
+ fi
+ else
+ # if we're here, then maven actually failed
+ modules_messages patch asflicense true
+ fi
+ return 1
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch.d/builtin-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/builtin-personality.sh b/dev-support/test-patch.d/builtin-personality.sh
new file mode 100644
index 0000000..dc944e4
--- /dev/null
+++ b/dev-support/test-patch.d/builtin-personality.sh
@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function builtin_personality_modules
+{
+ local repostatus=$1
+ local testtype=$2
+
+ local module
+
+ yetus_debug "Using builtin personality_modules"
+ yetus_debug "Personality: ${repostatus} ${testtype}"
+
+ clear_personality_queue
+
+ # this always makes sure the local repo has a fresh
+ # copy of everything per pom rules.
+ if [[ ${repostatus} == branch
+ && ${testtype} == mvninstall ]];then
+ personality_enqueue_module .
+ return
+ fi
+
+ for module in ${CHANGED_MODULES}; do
+ # shellcheck disable=SC2086
+ personality_enqueue_module ${module}
+ done
+}
+
+function personality_modules
+{
+ builtin_personality_modules "$@"
+}
+
+function builtin_mvn_personality_file_tests
+{
+ local filename=$1
+
+ yetus_debug "Using builtin mvn personality_file_tests"
+
+ if [[ ${filename} =~ src/main/webapp ]]; then
+ yetus_debug "tests/webapp: ${filename}"
+ elif [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ elif [[ ${filename} =~ \.md$
+ || ${filename} =~ \.md\.vm$
+ || ${filename} =~ src/site
+ || ${filename} =~ src/main/docs
+ ]]; then
+ yetus_debug "tests/site: ${filename}"
+ add_test site
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ src/test
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test javac
+ add_test mvninstall
+ add_test unit
+ elif [[ ${filename} =~ pom.xml$
+ || ${filename} =~ \.java$
+ || ${filename} =~ \.scala$
+ || ${filename} =~ src/main
+ ]]; then
+ if [[ ${filename} =~ src/main/bin
+ || ${filename} =~ src/main/sbin ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ else
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test javac
+ add_test javadoc
+ add_test mvninstall
+ add_test unit
+ fi
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
+
+function builtin_ant_personality_file_tests
+{
+ local filename=$1
+
+ yetus_debug "Using builtin ant personality_file_tests"
+
+ if [[ ${filename} =~ \.sh
+ || ${filename} =~ \.cmd
+ ]]; then
+ yetus_debug "tests/shell: ${filename}"
+ elif [[ ${filename} =~ \.c$
+ || ${filename} =~ \.cc$
+ || ${filename} =~ \.h$
+ || ${filename} =~ \.hh$
+ || ${filename} =~ \.proto$
+ || ${filename} =~ src/test
+ || ${filename} =~ \.cmake$
+ || ${filename} =~ CMakeLists.txt
+ ]]; then
+ yetus_debug "tests/units: ${filename}"
+ add_test javac
+ add_test unit
+ elif [[ ${filename} =~ build.xml
+ || ${filename} =~ ivy.xml
+ || ${filename} =~ \.java$
+ ]]; then
+ yetus_debug "tests/javadoc+units: ${filename}"
+ add_test javac
+ add_test javadoc
+ add_test unit
+ fi
+
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+}
+
+function builtin_personality_file_tests
+{
+ case ${BUILDTOOL} in
+ maven)
+ builtin_mvn_personality_file_tests "$@"
+ ;;
+ ant)
+ builtin_ant_personality_file_tests "$@"
+ ;;
+ *)
+ return 1
+ ;;
+ esac
+}
+
+function personality_file_tests
+{
+ builtin_personality_file_tests "$@"
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch.d/findbugs.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/findbugs.sh b/dev-support/test-patch.d/findbugs.sh
new file mode 100644
index 0000000..2fe2339
--- /dev/null
+++ b/dev-support/test-patch.d/findbugs.sh
@@ -0,0 +1,379 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+FINDBUGS_HOME=${FINDBUGS_HOME:-}
+FINDBUGS_WARNINGS_FAIL_PRECHECK=false
+
+add_plugin findbugs
+
+function findbugs_file_filter
+{
+ local filename=$1
+
+ if [[ ${BUILDTOOL} == maven
+ || ${BUILDTOOL} == ant ]]; then
+ if [[ ${filename} =~ \.java$ ]]; then
+ add_test findbugs
+ fi
+ fi
+}
+
+function findbugs_usage
+{
+ echo "FindBugs specific:"
+ echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME environment variable)"
+ echo "--findbugs-strict-precheck If there are Findbugs warnings during precheck, fail"
+}
+
+function findbugs_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --findbugs-home=*)
+ FINDBUGS_HOME=${i#*=}
+ ;;
+ --findbugs-strict-precheck)
+ FINDBUGS_WARNINGS_FAIL_PRECHECK=true
+ ;;
+ esac
+ done
+}
+
+## @description are the needed bits for findbugs present?
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 findbugs will work for our use
+## @return 1 findbugs is missing some component
+function findbugs_is_installed
+{
+ if [[ ! -e "${FINDBUGS_HOME}/bin/findbugs" ]]; then
+ printf "\n\n%s is not executable.\n\n" "${FINDBUGS_HOME}/bin/findbugs"
+ add_vote_table -1 findbugs "Findbugs is not installed."
+ return 1
+ fi
+ return 0
+}
+
+## @description Run the maven findbugs plugin and record found issues in a bug database
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function findbugs_runner
+{
+ local name=$1
+ local module
+ local result=0
+ local fn
+ local warnings_file
+ local i=0
+ local savestop
+
+ personality_modules "${name}" findbugs
+ case ${BUILDTOOL} in
+ maven)
+ modules_workers "${name}" findbugs clean test findbugs:findbugs
+ ;;
+ ant)
+ modules_workers "${name}" findbugs findbugs
+ ;;
+ esac
+
+ #shellcheck disable=SC2153
+ until [[ ${i} -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+ start_clock
+ offset_clock "${MODULE_STATUS_TIMER[${i}]}"
+ module="${MODULE[${i}]}"
+ fn=$(module_file_fragment "${module}")
+
+ case ${BUILDTOOL} in
+ maven)
+ file="${module}/target/findbugsXml.xml"
+ ;;
+ ant)
+ file="${ANT_FINDBUGSXML}"
+ ;;
+ esac
+
+ if [[ ! -f ${file} ]]; then
+ module_status ${i} -1 "" "${name}/${module} no findbugs output file (${file})"
+ ((i=i+1))
+ continue
+ fi
+
+ warnings_file="${PATCH_DIR}/${name}-findbugs-${fn}-warnings"
+
+ cp -p "${file}" "${warnings_file}.xml"
+
+ if [[ ${name} == branch ]]; then
+ "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name "${PATCH_BRANCH}" \
+ "${warnings_file}.xml" "${warnings_file}.xml"
+ else
+ "${FINDBUGS_HOME}/bin/setBugDatabaseInfo" -name patch \
+ "${warnings_file}.xml" "${warnings_file}.xml"
+ fi
+ if [[ $? != 0 ]]; then
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ module_status ${i} -1 "" "${name}/${module} cannot run setBugDatabaseInfo from findbugs"
+ ((retval = retval + 1))
+ ((i=i+1))
+ continue
+ fi
+
+ "${FINDBUGS_HOME}/bin/convertXmlToText" -html \
+ "${warnings_file}.xml" \
+ "${warnings_file}.html"
+ if [[ $? != 0 ]]; then
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ module_status ${i} -1 "" "${name}/${module} cannot run convertXmlToText from findbugs"
+ ((result = result + 1))
+ fi
+
+ if [[ -z ${FINDBUGS_VERSION}
+ && ${name} == branch ]]; then
+ FINDBUGS_VERSION=$(${GREP} -i "BugCollection version=" "${warnings_file}.xml" \
+ | cut -f2 -d\" \
+ | cut -f1 -d\" )
+ if [[ -n ${FINDBUGS_VERSION} ]]; then
+ add_footer_table findbugs "v${FINDBUGS_VERSION}"
+ fi
+ fi
+
+ ((i=i+1))
+ done
+ return ${result}
+}
+
+## @description Track pre-existing findbugs warnings
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function findbugs_preapply
+{
+ local fn
+ local module
+ local i=0
+ local warnings_file
+ local module_findbugs_warnings
+ local results=0
+
+ big_console_header "Pre-patch findbugs detection"
+
+ verify_needed_test findbugs
+
+ if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need findbugs tests."
+ return 0
+ fi
+
+ findbugs_is_installed
+ if [[ $? != 0 ]]; then
+ return 1
+ fi
+
+ findbugs_runner branch
+ results=$?
+
+ if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+ module=${MODULE[${i}]}
+ start_clock
+ offset_clock "${MODULE_STATUS_TIMER[${i}]}"
+ fn=$(module_file_fragment "${module}")
+ warnings_file="${PATCH_DIR}/branch-findbugs-${fn}-warnings"
+ # shellcheck disable=SC2016
+ module_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first \
+ "${PATCH_BRANCH}" \
+ "${warnings_file}.xml" \
+ "${warnings_file}.xml" \
+ | ${AWK} '{print $1}')
+
+ if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
+ module_status ${i} -1 "branch-findbugs-${fn}.html" "${module} in ${PATCH_BRANCH} cannot run convertXmlToText from findbugs"
+ ((results=results+1))
+ fi
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ ((i=i+1))
+ done
+ modules_messages branch findbugs true
+ fi
+
+ if [[ ${results} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
+
+## @description Verify patch does not trigger any findbugs warnings
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return 0 on success
+## @return 1 on failure
+function findbugs_postinstall
+{
+ local module
+ local fn
+ local combined_xml
+ local branchxml
+ local patchxml
+ local newbugsbase
+ local new_findbugs_warnings
+ local line
+ local firstpart
+ local secondpart
+ local i=0
+ local results=0
+ local savestop
+
+ big_console_header "Patch findbugs detection"
+
+ verify_needed_test findbugs
+
+ if [[ $? == 0 ]]; then
+ echo "Patch does not appear to need findbugs tests."
+ return 0
+ fi
+
+ findbugs_is_installed
+ if [[ $? != 0 ]]; then
+ return 1
+ fi
+
+ findbugs_runner patch
+
+ until [[ $i -eq ${#MODULE[@]} ]]; do
+ if [[ ${MODULE_STATUS[${i}]} == -1 ]]; then
+ ((result=result+1))
+ ((i=i+1))
+ continue
+ fi
+ start_clock
+ offset_clock "${MODULE_STATUS_TIMER[${i}]}"
+ module="${MODULE[${i}]}"
+ pushd "${module}" >/dev/null
+ fn=$(module_file_fragment "${module}")
+
+ combined_xml="${PATCH_DIR}/combined-findbugs-${fn}.xml"
+ branchxml="${PATCH_DIR}/branch-findbugs-${fn}-warnings.xml"
+ patchxml="${PATCH_DIR}/patch-findbugs-${fn}-warnings.xml"
+
+ if [[ ! -f "${branchxml}" ]]; then
+ branchxml=${patchxml}
+ fi
+
+ newbugsbase="${PATCH_DIR}/new-findbugs-${fn}"
+
+ "${FINDBUGS_HOME}/bin/computeBugHistory" -useAnalysisTimes -withMessages \
+ -output "${combined_xml}" \
+ "${branchxml}" \
+ "${patchxml}"
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ module_status ${i} -1 "" "${module} cannot run computeBugHistory from findbugs"
+ ((result=result+1))
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ ((i=i+1))
+ continue
+ fi
+
+ #shellcheck disable=SC2016
+ new_findbugs_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -first patch \
+ "${combined_xml}" "${newbugsbase}.xml" | ${AWK} '{print $1}')
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ module_status ${i} -1 "" "${module} cannot run filterBugs (#1) from findbugs"
+ ((result=result+1))
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ ((i=i+1))
+ continue
+ fi
+
+ #shellcheck disable=SC2016
+ new_findbugs_fixed_warnings=$("${FINDBUGS_HOME}/bin/filterBugs" -fixed patch \
+ "${combined_xml}" "${newbugsbase}.xml" | ${AWK} '{print $1}')
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ module_status ${i} -1 "" "${module} cannot run filterBugs (#2) from findbugs"
+ ((result=result+1))
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ ((i=i+1))
+ continue
+ fi
+
+ echo "Found ${new_findbugs_warnings} new Findbugs warnings and ${new_findbugs_fixed_warnings} newly fixed warnings."
+ findbugs_warnings=$((findbugs_warnings+new_findbugs_warnings))
+ findbugs_fixed_warnings=$((findbugs_fixed_warnings+new_findbugs_fixed_warnings))
+
+ "${FINDBUGS_HOME}/bin/convertXmlToText" -html "${newbugsbase}.xml" \
+ "${newbugsbase}.html"
+ if [[ $? != 0 ]]; then
+ popd >/dev/null
+ module_status ${i} -1 "" "${module} cannot run convertXmlToText from findbugs"
+ ((result=result+1))
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ ((i=i+1))
+ continue
+ fi
+
+ if [[ ${new_findbugs_warnings} -gt 0 ]] ; then
+ populate_test_table FindBugs "module:${module}"
+ while read line; do
+ firstpart=$(echo "${line}" | cut -f2 -d:)
+ secondpart=$(echo "${line}" | cut -f9- -d' ')
+ add_test_table "" "${firstpart}:${secondpart}"
+ done < <("${FINDBUGS_HOME}/bin/convertXmlToText" "${newbugsbase}.xml")
+
+ module_status ${i} -1 "new-findbugs-${fn}.html" "${module} introduced "\
+ "${new_findbugs_warnings} new FindBugs issues."
+ ((result=result+1))
+ fi
+ savestop=$(stop_clock)
+ MODULE_STATUS_TIMER[${i}]=${savestop}
+ popd >/dev/null
+ ((i=i+1))
+ done
+
+ modules_messages patch findbugs true
+ if [[ ${result} != 0 ]]; then
+ return 1
+ fi
+ return 0
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37578379/dev-support/test-patch.d/github.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.d/github.sh b/dev-support/test-patch.d/github.sh
new file mode 100644
index 0000000..281f15b
--- /dev/null
+++ b/dev-support/test-patch.d/github.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_bugsystem github
+
+## @description Write the contents of a file to github
+## @params filename
+## @stability stable
+## @audience public
+function github_write_comment
+{
+ local -r commentfile=${1}
+ shift
+
+ local retval=1
+
+ return ${retval}
+}
+
+
+## @description Print out the finished details to the Github PR
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param runresult
+function github_finalreport
+{
+ local result=$1
+ local i
+ local commentfile=${PATCH_DIR}/commentfile
+ local comment
+
+ rm "${commentfile}" 2>/dev/null
+
+ if [[ ${JENKINS} != "true" ]] ; then
+ return 0
+ fi
+
+ big_console_header "Adding comment to Github"
+
+ add_footer_table "Console output" "${BUILD_URL}console"
+
+ if [[ ${result} == 0 ]]; then
+ add_header_line ":confetti_ball: **+1 overall**"
+ else
+ add_header_line ":broken_heart: **-1 overall**"
+ fi
+
+ printf "\n\n\n\n" >> "${commentfile}"
+
+ i=0
+ until [[ $i -eq ${#TP_HEADER[@]} ]]; do
+ printf "%s\n\n" "${TP_HEADER[${i}]}" >> "${commentfile}"
+ ((i=i+1))
+ done
+
+ {
+ printf "\n\n"
+ echo "| Vote | Subsystem | Runtime | Comment |"
+ echo "|:----:|----------:|--------:|:--------|"
+ } >> "${commentfile}"
+
+ i=0
+ until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
+ echo "${TP_VOTE_TABLE[${i}]}" >> "${commentfile}"
+ ((i=i+1))
+ done
+
+ if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
+ {
+ printf "\n\n"
+ echo "| Reason | Tests |"
+ echo "|-------:|:------|"
+ } >> "${commentfile}"
+ i=0
+ until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do
+ echo "${TP_TEST_TABLE[${i}]}" >> "${commentfile}"
+ ((i=i+1))
+ done
+ fi
+
+ {
+ printf "\n\n"
+ echo "| Subsystem | Report/Notes |"
+ echo "|----------:|:-------------|"
+ } >> "${commentfile}"
+
+ i=0
+ until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
+ comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
+ ${SED} -e "s,@@BASE@@,${BUILD_URL}artifact/patchprocess,g")
+ printf "%s\n" "${comment}" >> "${commentfile}"
+ ((i=i+1))
+ done
+
+ printf "\n\nThis message was automatically generated.\n\n" >> "${commentfile}"
+
+ write_to_github "${commentfile}"
+}