You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2018/12/13 16:11:35 UTC
[1/2] yetus git commit: Comprehensive CI system patch
Repository: yetus
Updated Branches:
refs/heads/master e94d8c4b3 -> 492a20e05
Comprehensive CI system patch
YETUS-681. Add robots plugin type
YETUS-649. Travis CI support
YETUS-684. add support for Gitlab CI
YETUS-687. add better docker env support
+ Circle CI support
+ --git-offline
+ --git-shallow
+ autodetect build tool
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/60745a0d
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/60745a0d
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/60745a0d
Branch: refs/heads/master
Commit: 60745a0d9358e30fc725709db57a32193a024e98
Parents: e94d8c4
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Oct 29 09:39:32 2018 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Dec 13 08:05:25 2018 -0800
----------------------------------------------------------------------
.circleci/config.yml | 52 ++++++
.gitlab-ci.yml | 33 ++++
.travis.yml | 34 ++++
Jenkinsfile | 120 +++++++++++++
.../source/documentation/in-progress.html.md | 2 +
.../in-progress/precommit-advanced.md | 6 +-
.../in-progress/precommit-basic.md | 130 +++++++-------
.../in-progress/precommit-bugsystems.md | 23 ++-
.../in-progress/precommit-robots.md | 115 ++++++++++++
precommit/src/main/shell/core.d/01-common.sh | 76 +++++++-
precommit/src/main/shell/core.d/docker.sh | 37 +++-
precommit/src/main/shell/core.d/patchfiles.sh | 4 +
precommit/src/main/shell/robots.d/circleci.sh | 72 ++++++++
precommit/src/main/shell/robots.d/gitlabci.sh | 52 ++++++
precommit/src/main/shell/robots.d/jenkins.sh | 167 +++++++++++++++++
precommit/src/main/shell/robots.d/travisci.sh | 90 ++++++++++
.../src/main/shell/test-patch-docker/Dockerfile | 3 +
precommit/src/main/shell/test-patch.d/github.sh | 30 ++--
precommit/src/main/shell/test-patch.d/gitlab.sh | 52 ++++--
precommit/src/main/shell/test-patch.d/maven.sh | 13 +-
precommit/src/main/shell/test-patch.sh | 180 +++++++++++--------
21 files changed, 1111 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/.circleci/config.yml
----------------------------------------------------------------------
diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 0000000..d225443
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+version: 2
+jobs:
+ build:
+ docker:
+ - image: apache/yetus:master
+
+ working_directory: ~/repo
+
+ environment:
+ # Customize the JVM maximum heap limit
+ MAVEN_OPTS: -Xmx3200m
+
+ steps:
+ - checkout
+
+ # Download and cache dependencies
+ - restore_cache:
+ keys:
+ - v1-dependencies-{{ checksum "pom.xml" }}
+ # fallback to using the latest cache if no exact match is found
+ - v1-dependencies-
+
+ - run: >
+ ~/repo/precommit/src/main/shell/test-patch.sh
+ --plugins=all
+ --patch-dir=/tmp/yetus-out
+ --tests-filter=checkstyle,javadoc,rubocop,ruby-lint,test4tests
+
+ - save_cache:
+ paths:
+ - ~/.m2
+ key: v1-dependencies-{{ checksum "pom.xml" }}
+
+ - store_artifacts:
+ path: /tmp/yetus-out
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/.gitlab-ci.yml
----------------------------------------------------------------------
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..52e6228
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+buretoolbox-job:
+ image: apache/yetus:master
+ allow_failure: true
+ script:
+ - >
+ precommit/src/main/shell/test-patch.sh
+ --patch-dir=/tmp/yetus-out
+ --plugins=all
+ --java-home=/usr/lib/jvm/java-8-openjdk-amd64
+ --tests-filter=checkstyle,javadoc,rubocop,ruby-lint,test4tests
+
+ artifacts:
+ expire_in: 1 week
+ when: always
+ paths:
+ - yetus-out/
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..9e10bde
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+sudo: required
+
+services:
+ - docker
+
+addons:
+ artifacts: true
+ paths:
+ - yetus-out
+
+script:
+ - >
+ precommit/src/main/shell/test-patch.sh
+ --docker
+ --patch-dir=/tmp/yetus-out
+ --plugins=all
+ --tests-filter=checkstyle,javadoc,rubocop,ruby-lint,test4tests
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/Jenkinsfile
----------------------------------------------------------------------
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000..04396eb
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+pipeline {
+ agent {
+ // Hadoop and ubuntu for ASF, rest are private
+ label 'Hadoop||ubuntu||azaka||small'
+ }
+ triggers {
+ cron('@daily')
+ }
+ options {
+ buildDiscarder(logRotator(numToKeepStr: '5'))
+ timeout (time: 9, unit: 'HOURS')
+ timestamps()
+ checkoutToSubdirectory('src')
+ }
+ environment {
+ //YETUS_RELEASE = '0.8.0'
+ YETUS_BASEDIR = 'src'
+ // will also need to change email section below
+ YETUS_RELATIVE_PATCHDIR = 'out'
+ YETUS_DOCKERFILE = "${YETUS_BASEDIR}/precommit/src/main/shell/test-patch-docker/Dockerfile"
+ }
+ stages {
+ stage ('precommit-run') {
+ steps {
+ sh '''#!/usr/bin/env bash
+ if [[ -d "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}" ]]; then
+ rm -rf "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
+ fi
+ mkdir -p "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
+
+ # where the source is located
+ YETUS_ARGS+=("--basedir=${WORKSPACE}/${YETUS_BASEDIR}")
+
+ # nuke the src repo before working
+ YETUS_ARGS+=("--resetrepo")
+
+ # Enable maven custom repos in order to avoid multiple executor clashes
+ YETUS_ARGS+=("--mvn-custom-repos")
+
+ # run in docker mode
+ YETUS_ARGS+=("--docker")
+
+ # temp storage, etc
+ YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}")
+
+ # lots of different output formats
+ YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/brief.txt")
+ YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/console.txt")
+ YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/report.html")
+
+ # rsync these files back into the archive dir
+ YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+
+ # URL for user-side presentation
+ YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+
+ # plugins to enable
+ YETUS_ARGS+=("--plugins=all")
+
+ YETUS_ARGS+=("--tests-filter=checkstyle,javadoc,rubocop,ruby-lint,tests4tests")
+
+ # run test-patch from the source tree specified up above
+ TESTPATCHBIN=${WORKSPACE}/src/precommit/src/main/shell/test-patch.sh
+
+ /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+
+ '''
+ }
+ }
+ }
+ post {
+ always {
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts "${env.YETUS_RELATIVE_PATCHDIR}/**"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "${env.YETUS_RELATIVE_PATCHDIR}",
+ reportFiles: 'report.html',
+ reportName: 'Yetus QBT Report'
+ ]
+ }
+ failure {
+ emailext subject: '$DEFAULT_SUBJECT',
+ body: '''For more details, see ${BUILD_URL}
+
+${CHANGES, format="[%d] (%a) %m"}
+
+${FILE,path="out/brief.txt"}''',
+ recipientProviders: [
+ [$class: 'CulpritsRecipientProvider'],
+ [$class: 'DevelopersRecipientProvider'],
+ [$class: 'RequesterRecipientProvider']
+ ],
+ replyTo: '$DEFAULT_REPLYTO',
+ to: '$DEFAULT_RECIPIENTS'
+ }
+ cleanup() {
+ deleteDir()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/asf-site-src/source/documentation/in-progress.html.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress.html.md b/asf-site-src/source/documentation/in-progress.html.md
index b980012..526a2d8 100644
--- a/asf-site-src/source/documentation/in-progress.html.md
+++ b/asf-site-src/source/documentation/in-progress.html.md
@@ -29,6 +29,8 @@ The Yetus Precommit Patch Tester allows projects to codify their patch acceptanc
For a complete guide to the Precommit API, see [the generated API documentation](precommit-apidocs/).
+For help with CI tools and automation, see our documentation about [robots](precommit-robots).
+
# Yetus Release Doc Maker
The Release Documentation Maker allows projects to generate nicely formated Markdown Changelogs and Release Notes based upon JIRA. You can view that
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/asf-site-src/source/documentation/in-progress/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-advanced.md b/asf-site-src/source/documentation/in-progress/precommit-advanced.md
index bdfe5fe..a2fe146 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-advanced.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-advanced.md
@@ -28,7 +28,7 @@ test-patch
# Docker Support
-By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself in a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a Jenkins instance.
+By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself in a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a CI instance.
The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. test-patch will copy this file up until the text "YETUS CUT HERE" to a different directory and then append its necessary hooks to re-launch itself prior to executing docker.
@@ -273,7 +273,7 @@ This list is used if the user does not provide a list of plug-ins.
There are a handful of extremely important system variables that make life easier for personality and plug-in writers. Other variables may be provided by individual plug-ins. Check their development documentation for more information.
-* BUILD\_NATIVE will be set to true if the system has requested that non-JVM-based code be built (e.g., JNI or other compiled C code). Under Jenkins, this is always true.
+* BUILD\_NATIVE will be set to true if the system has requested that non-JVM-based code be built (e.g., JNI or other compiled C code). For [robots](precommit-robots), this is always true.
* BUILDTOOL specifies which tool is currently being used to drive compilation. Additionally, many build tools define xyz\_ARGS to pass on to the build tool command line. (e.g., MAVEN\_ARGS if maven is in use). Projects may set this in their personality. NOTE: today, only one build tool at a time is supported. This may change in the future.
@@ -295,4 +295,4 @@ There are a handful of extremely important system variables that make life easie
* PATCH\_NAMING\_RULE should be a URL that points to a project's on-boarding documentation for new users. It is used to suggest a review of patch naming guidelines. Since this should be project specific information, it is useful to set in a project's personality.
-* TEST\_PARALLEL if parallel unit tests have been requested. Project personalities are responsible for actually enabling or ignoring the request. TEST\_THREADS is the number of threads that have been requested to run in parallel.
+* TEST\_PARALLEL if parallel unit tests have been requested. Project personalities are responsible for actually enabling or ignoring the request. TEST\_THREADS is the number of threads that have been requested to run in parallel. For [robots](precommit-robots), this is always true.
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/asf-site-src/source/documentation/in-progress/precommit-basic.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-basic.md b/asf-site-src/source/documentation/in-progress/precommit-basic.md
index ab55c15..fe3348d 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-basic.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-basic.md
@@ -23,7 +23,7 @@ test-patch
* [Purpose](#purpose)
* [Pre-requisites](#pre-requisites)
* [Basic Usage](#basic-usage)
-* [Automation](#automation)
+* [Output Directory](#output-directory)
* [Build Tool](#build-tool)
* [Providing Patch Files](#providing-patch-files)
* [Project-Specific Capabilities](#project-specific-capabilities)
@@ -33,7 +33,7 @@ test-patch
# Purpose
-As part of Apache Hadoop's commit process, all patches to the source base go through a precommit test that does some (relatively) light checking to make sure the proposed change does not break unit tests and/or passes some other prerequisites such as code formatting guidelines. This is meant as a preliminary check for committers so that the basic patch is in a known state and for contributors to know if they have followed the project's guidelines. This check, called test-patch, along with a helper program, called smart-apply-patch, may also be used by individual developers to verify a patch prior to sending to the Apache Hadoop QA systems.
+As part of Apache Hadoop's commit process, all patches to the source base go through a precommit test that does some (relatively) light checking to make sure the proposed change does not break unit tests and/or passes some other prerequisites such as code formatting guidelines. This is meant as a preliminary check for committers so that the basic patch is in a known state and for contributors to know if they have followed the project's guidelines. This check, called `test-patch`, along with a helper program, called `smart-apply-patch`, may also be used by individual developers to verify a patch prior to sending to the Apache Hadoop QA systems.
Other projects have adopted a similar methodology after seeing great success in the Apache Hadoop model. Some have even gone as far as forking Apache Hadoop's precommit code and modifying it to meet their project's needs.
@@ -43,9 +43,9 @@ as a whole.
# Pre-requisites
-test-patch and smart-apply-patch are written in bash for maximum portability. As such, it mostly assumes the locations of commands to be in the file path. However, in many cases, this assumption may be overridden via command line options.
+`test-patch` and `smart-apply-patch` are written in bash for maximum portability. As such, it mostly assumes the locations of commands to be in the file path. However, in many cases, this assumption may be overridden via command line options.
-For Solaris and Solaris-like operating systems, the default location for the POSIX binaries is in /usr/xpg4/bin and the default location for the GNU binaries is /usr/gnu/bin.
+For Solaris and Solaris-like operating systems, the default location for the POSIX binaries is in `/usr/xpg4/bin` and the default location for the GNU binaries is `/usr/gnu/bin`.
## Base Requirements
@@ -70,6 +70,7 @@ Bug Systems:
* [GitHub](https://github.com/)-based issue tracking
* [JIRA](https://www.atlassian.com/software/jira)-based issue tracking
* [Bugzilla](https://www.bugzilla.org/)-based issue tracking (Read Only)
+* [Gitlab](https://www.gitlab.com)-based issue tracking
Build Tools:
@@ -82,8 +83,11 @@ Build Tools:
Automation and Isolation:
+* [Circle CI](https://www.circleci.com)
* [Docker](https://www.docker.com) version 1.6.0+
+* [Gitlab CI](https://www.gitlab.com)
* [Jenkins](https://www.jenkins-ci.org)
+* [Travis CI](https://www.travis-ci.com)
Unit Test Formats:
@@ -96,6 +100,7 @@ Language Support, Licensing, and more:
* [Apache Creadur Rat](http://creadur.apache.org/rat/) entries in build system
* [checkstyle](http://checkstyle.sourceforge.net/) entries in build system (ant and maven only)
* [FindBugs](http://findbugs.sourceforge.net/) entries in build system and 3.x executables
+ (NOTE: FindBugs executables are required even if the build system is using [Spotbugs](https://spotbugs.github.io/))
* [Perl::Critic](http://perlcritic.com/) installed
* [pylint](http://www.pylint.org/) installed
* [rubocop](http://batsov.com/rubocop/) installed
@@ -107,25 +112,25 @@ Language Support, Licensing, and more:
The first step for a successful deployment is determining which features/plug-ins to enable:
```bash
-$ test-patch.sh --list-plugins
+$ test-patch --list-plugins
```
This option will list all of the available plug-ins that are installed in the default location. From this list, the specific plug-ins can be enabled:
```bash
-$ test-patch.sh --plugins="ant,maven,shellcheck,xml" <other options>
+$ test-patch --plugins="ant,maven,shellcheck,xml" <other options>
```
As a short-cut, every plug-in may be enabled via the special 'all' type:
```bash
-$ test-patch.sh --plugins="all" <other options>
+$ test-patch --plugins="all" <other options>
```
`--plugins` also allows some basic "arithmetic":
```bash
-$ test-patch.sh --plugins="all,-checkstyle,-findbugs" <other options>
+$ test-patch --plugins="all,-checkstyle,-findbugs" <other options>
```
This will enable all plug-ins for potential usage, except for checkstyle and findbugs.
@@ -136,17 +141,16 @@ This command will execute basic patch testing against a patch file stored in "fi
```bash
$ cd <your repo>
-$ test-patch.sh --dirty-workspace --project=projectname <filename>
+$ test-patch --dirty-workspace --project=projectname <filename>
```
The `--dirty-workspace` flag tells test-patch that the repository is not clean and it is ok to continue. By default, unit tests are not run since they may take a significant amount of time.
To do turn them on, we need to provide the --run-tests option:
-
```bash
$ cd <your repo>
-$ test-patch.sh --dirty-workspace --run-tests <filename>
+$ test-patch --dirty-workspace --run-tests <filename>
```
This is the same command, but now runs the unit tests.
@@ -157,14 +161,14 @@ A typical configuration is to have two repositories. One with the code you are
$ cd <workrepo>
$ git diff master > /tmp/patchfile
$ cd ../<testrepo>
-$ test-patch.sh --basedir=<testrepo> --resetrepo /tmp/patchfile
+$ test-patch --basedir=<testrepo> --resetrepo /tmp/patchfile
```
-We used two new options here. --basedir sets the location of the repository to use for testing. --resetrepo tells test patch that it can go into **destructive** mode. Destructive mode will wipe out any changes made to that repository, so use it with care!
+We used two new options here. `--basedir` sets the location of the repository to use for testing. `--resetrepo` tells test patch that it can go into **destructive** mode. Destructive mode will wipe out any changes made to that repository, so use it with care!
# Fork Bomb Protection
-By default, test-patch.sh will set the user soft limit (ulimit -Su) to a relatively low 1,000 processes (and, on some operating systems with some languages such as Java, threads!). This is to prevent errant processes from eating up all system resources. If this limit is too low, it may be necessary to use the `--proclimit` option. For example:
+By default, `test-patch` will set the user soft limit (`ulimit -Su`) to a relatively low 1,000 processes (and, on some operating systems with some languages such as Java, threads!). This is to prevent errant processes from eating up all system resources. If this limit is too low, it may be necessary to use the `--proclimit` option. For example:
```bash
$ test-patch --proclimit=10000
@@ -174,68 +178,51 @@ $ test-patch --proclimit=10000
NOTE: The actual implementation of this feature is dependent upon the version of Bash. For bash v4 and higher (most operating systems), the fork bomb protection is generally only used for the build and QA tools. This means Apache Yetus should continue to function. For earlier versions of bash (e.g., OS X), the limit is applied to all of test-patch. If the limit is hit, Apache Yetus will itself likely crash.
-# Automation
+# Output Directory
-After the tests have run, there is a directory that contains all of the test-patch related artifacts. This is generally referred to as the patchprocess directory. By default, test-patch tries to make something off of /tmp to contain this content. Using the `--patch-dir` option, one can specify exactly which directory to use. This is helpful for automated precommit testing so that Jenkins or other automated workflow system knows where to look to gather up the output.
+After the tests have run, there is a directory that contains all of the `test-patch` related artifacts. This is generally referred to as the patch directory. By default, `test-patch` tries to make something off of /tmp to contain this content. Using the `--patch-dir` option, one can specify exactly which directory to use. This is helpful for automated precommit testing so that [continuous integration systems](../precommit-robots) knows where to look to gather up the output.
For example:
```bash
-$ test-patch.sh --robot --patch-dir=${WORKSPACE}/patchprocess --basedir=${WORKSPACE}/source ${WORKSPACE}/patchfile
+$ test-patch --patch-dir=${WORKSPACE}/patchdir --basedir=${WORKSPACE}/source ${WORKSPACE}/patchfile
```
-... will trigger test-patch to run in fully automated mode, using ${WORKSPACE}/patchprocess as its scratch space, ${WORKSPACE}/source as the source repository, and ${WORKSPACE}/patchfile as the name of the patch to test against. This will always run the unit tests, write answers back to bug systems, remove old, stopped/exited Docker containers after 24 hours and images after 1 week, forcibly use --resetrepo, and more.
+... will trigger `test-patch` to run in fully automated mode, using `${WORKSPACE}/patchdir` as its scratch space, `${WORKSPACE}/source` as the source repository, and `${WORKSPACE}/patchfile` as the name of the patch to test against. This will always run the unit tests, write answers back to bug systems, remove old, stopped/exited Docker containers after 24 hours and images after 1 week, forcibly use `--resetrepo`, and more.
**NOTE: Make sure to add the patch directory to `.gitignore` if the directory is inside the source tree to avoid deleting it, as `test-patch` does a `git clean` to remove untracked files from previous runs.**
-The --build-url option is also useful when running in --robot mode so that emails and such
-have a location to look at the output artifacts:
-
-```bash
-$ test-patch.sh --robot --build-url=http://server.example.name:80/${buildnumber}/
-```
-
-Some plug-ins such as Maven have special handling if there are multiple executions of test-patch happening at once. It is very common when using automation systems to have multiple runs on the same host. In order to assist these plug-ins, an instance identifier may be provided:
-
-```bash
-$ test-patch.sh --robot --instance=1
-```
-
-If --robot is specified without an instance, a random number is generated and used.
+# Build Tool
-There is some special handling if Jenkins is actually your automation tool. Instead of using --robot, use --jenkins:
+Out of the box, test-patch will try to figure out which build tool the project uses. But what if you want to override it? The `--build-tool` option allows a manual setting:
```bash
-$ test-patch.sh --jenkins --patch-dir=${WORKSPACE}/patchprocess --basedir=${WORKSPACE}/source ${WORKSPACE}/patchfile
+$ test-patch (other options) --build-tool=ant
```
-This will enable --robot, set the --build-url option from the ${BUILD_URL} environment variable, and the instance identifier is set to the ${EXECUTOR_NUMBER}.
-
-If stuck containers are a problem, a more aggressive robot may be enabled with the --sentinel option. This option enables killing containers that have been running for over 24 hours as well.
-
-# Build Tool
+will tell `test-patch` to use `ant` instead of maven to drive the project.
-Out of the box, test-patch is built to use maven. But what if the project is built using something else, such as ant?
+To disable the build tool entirely, use the `nobuild` setting:
```bash
-$ test-patch.sh (other options) --build-tool=ant
+$ test-patch (other options) --build-tool=nobuild
```
-will tell test-patch to use ant instead of maven to drive the project.
-
# Providing Patch Files
+NOTE: More in-depth information may be found in the [bugsystems](../precommit-bugsystems/) section.
+
## JIRA
It is a fairly common practice within the Apache community to use Apache's JIRA instance to store potential patches. As a result, test-patch supports providing just a JIRA issue number. test-patch will find the *last* attachment, download it, then process it.
-**NOTE: test-patch expects the patch files to follow a particular naming convention. For complete details
+**NOTE: `test-patch` expects the patch files to follow a particular naming convention. For complete details
on the naming convention please refer to [patch-naming-conventions](../precommit-patchnames/)**
For example:
```bash
-$ test-patch.sh (other options) HADOOP-9905
+$ test-patch (other options) HADOOP-9905
```
... will process the patch file associated with this JIRA issue.
@@ -243,33 +230,56 @@ $ test-patch.sh (other options) HADOOP-9905
If the Apache JIRA system is not in use, then override options may be provided on the command line to point to a different JIRA instance.
```bash
-$ test-patch.sh --jira-issue-re='^PROJECT-[0-9]+$' --jira-base-url='https://example.com/jira' PROJECT-90
+$ test-patch --jira-issue-re='^PROJECT-[0-9]+$' --jira-base-url='https://example.com/jira' PROJECT-90
```
... will process the patch file attached to PROJECT-90 on the JIRA instance located on the example.com server.
## GITHUB
-test-patch has some basic support for Github. test-patch supports many forms of providing pull requests to work on:
+`test-patch` has built-in support for Github. `test-patch` supports many forms of providing pull requests to work on:
```bash
-$ test-patch.sh --github-repo=apache/pig GH:99
+$ test-patch --github-repo=apache/pig GH:99
```
or
```bash
-$ test-patch.sh https://github.com/apache/pig/pulls/99
+$ test-patch https://github.com/apache/pig/pulls/99
```
or
```bash
-$ test-patch.sh https://github.com/apache/pig/pulls/99.patch
+$ test-patch https://github.com/apache/pig/pulls/99.patch
```
... will process PR #99 on the apache/pig repo.
+## GITLAB
+
+`test-patch` has support for Gitlab. `test-patch` supports many forms of providing merge requests to work on:
+
+```bash
+$ test-patch --gitlab-repo=_a__w_/yetus GL:1
+```
+
+or
+
+```bash
+$ test-patch https://gitlab.com/_a__w_/yetus/merge_requests/3
+```
+
+or
+
+```bash
+$ test-patch https://gitlab.com/_a__w_/yetus/merge_requests/3.patch
+```
+
+... will process MR #3 on the \_a\_\_w\_/yetus repo.
+
+
## Generic URLs
Luckily, test-patch supports ways to provide unified diffs via URLs.
@@ -277,29 +287,29 @@ Luckily, test-patch supports ways to provide unified diffs via URLs.
For example:
```bash
-$ test-patch.sh (other options) https://example.com/webserver/file.patch
+$ test-patch (other options) https://example.com/webserver/file.patch
```
... will download and process the file.patch from the example.com webserver.
# Project-specific Capabilities
-Due to the extensible nature of the system, test-patch allows for projects to define project-specific rules which we call personalities. (How to build those rules is covered elsewhere.) There are two ways to specify which personality to use:
+Due to the extensible nature of the system, `test-patch` allows for projects to define project-specific rules which we call personalities. (How to build those rules is covered elsewhere.) There are two ways to specify which personality to use:
## Direct Method
```bash
-$ test-patch.sh (other options) --personality=(filename)
+$ test-patch (other options) --personality=(filename)
```
This tells test-patch to use the personality in the given file.
## Project Method
-However, test-patch can detect if it is a personality that is in its "personality" directory based upon the project name:
+However, `test-patch` can detect if it is a personality that is in its "personality" directory based upon the project name:
```bash
-$ test-patch.sh (other options) --project=(project)
+$ test-patch (other options) --project=(project)
```
# MultiJDK
@@ -307,23 +317,23 @@ $ test-patch.sh (other options) --project=(project)
For many projects, it is useful to test Java code against multiple versions of JDKs at the same time. test-patch can do this with the --multijdkdirs option:
```bash
-$ test-patch.sh (other options) --multijdkdirs="/j/d/k/1,/j/d/k/2"
+$ test-patch (other options) --multijdkdirs="/j/d/k/1,/j/d/k/2"
```
-Not all Java tests support this mode, but those that do will now run their tests with all of the given versions of Java consecutively (e.g., javac--the Java compliation test). Tests that do not support MultiJDK mode (e.g., checkstyle, mvn install) will use JAVA\_HOME.
+Not all Java tests support this mode, but those that do will now run their tests with all of the given versions of Java consecutively (e.g., `javac`--the Java compliation test). Tests that do not support MultiJDK mode (e.g., checkstyle, mvn install) will use JAVA\_HOME.
NOTE: JAVA\_HOME is always appended to the list of JDKs in MultiJDK mode. If JAVA\_HOME is in the list, it will be moved to the end.
# Docker
-test-patch also has a mode to utilize Docker:
+`test-patch` also has a mode to utilize Docker:
```bash
-$ test-patch.sh (other options) --docker
+$ test-patch. (other options) --docker
```
This will do some preliminary setup and then re-execute itself inside a Docker container. For more information on how to provide a custom Dockerfile and other Docker-specific features, see the advanced guide.
# In Closing
-test-patch has many other features and command line options for the basic user. Many of these are self-explanatory. To see the list of options, run test-patch.sh without any options or with --help.
+test-patch has many other features and command line options for the basic user. Many of these are self-explanatory. To see the list of options, run `test-patch` without any options or with --help.
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/asf-site-src/source/documentation/in-progress/precommit-bugsystems.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-bugsystems.md b/asf-site-src/source/documentation/in-progress/precommit-bugsystems.md
index b3b1403..757a44d 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-bugsystems.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-bugsystems.md
@@ -55,7 +55,7 @@ add_bugsystem <pluginname>
Currently, Bugzilla support is read-only. To use it, the Bug ID must be preferenced with 'BZ:'. For example:
```bash
-$ test-patch.sh (other options) BZ:4
+$ test-patch (other options) BZ:4
```
... will pull down Bugzilla ID #4.
@@ -64,7 +64,7 @@ Using the `--bugzilla-base-url` on the command line or BUGZILLA\_BASE\_URL in a
# GitHub Specific
-GitHub supports the full range of functionality, including putting comments on individual lines. Be aware, however, that test-patch.sh will (generally) require that GitHub PRs be fully rebased (i.e., a single commit) in many circumstances.
+GitHub supports the full range of functionality, including putting comments on individual lines. Be aware, however, that `test-patch` will (generally) require that GitHub PRs be fully squashed and rebased (i.e., a single commit) in many circumstances.
By default, the GitHub plug-in assumes that https://github.com is the base URL for GitHub. Enterprise users may override this with the `--github-base-url` for the normal web user interface and `--github-api-url` for the API URL. Personalities may use GITHUB\_API\_URL and GITHUB\_BASE\_URL.
@@ -87,6 +87,25 @@ GitHub pull requests may be directly processed on the command line in two ways:
The GitHub bugsystem plugin will attempt to download the unified diff that the pull request references.
Pull requests that are made off of a specific branch will switch the test repo to that branch, if permitted. If the pull request references a JIRA issue that matches the given JIRA issue regexp in the Subject, the JIRA plug-in will also be invoked as needed.
+# GitLab Specific
+
+GitLab supports the full range of functionality, including putting comments on individual lines. Be aware, however, that `test-patch` will (generally) require that GitLab MRs be fully squashed and rebased (i.e., a single commit) in many circumstances.
+
+By default, the GitLab plug-in assumes that https://gitlab.com is the base URL for GitHub. Enterprise users may override this with the `--gitlab-base-url` for the normal web user interface and `--gitlab-api-url` for the API URL. Personalities may use GITLAB\_API\_URL and GITLAB\_BASE\_URL.
+
+The specific repository on GitLab is defined with either `--gitlab-repo` on the command line or GITLAB\_REPO in a personality. It should take the form of "user/repo".
+
+In order to comment on issues or, depending upon the security setup of the repo, authentication credentials. The GitLab plug-in supports tokens via the `--gitlab-token` option or GITLAB\_TOKEN environment variable.
+
+
+GitLab merge requests may be directly processed on the command line in two ways:
+
+ * GL:(MR number)
+ * GLSHA:(MR SHA1 number)
+
+The GitLab bugsystem plugin will attempt to download the unified diff that the merge request references.
+Merge requests that are made off of a specific branch will switch the test repo to that branch, if permitted. If the merge request references a JIRA issue that matches the given JIRA issue regexp in the Subject, the JIRA plug-in will also be invoked as needed.
+
# JIRA Specific
JIRA support allows both patch downloads and summary writes. It also supports branch detection-based upon the name of the attached patch file.
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/asf-site-src/source/documentation/in-progress/precommit-robots.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.md b/asf-site-src/source/documentation/in-progress/precommit-robots.md
new file mode 100644
index 0000000..b0fab18
--- /dev/null
+++ b/asf-site-src/source/documentation/in-progress/precommit-robots.md
@@ -0,0 +1,115 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+Robots: Continuous Integration Support
+======================================
+
+`test-patch` works hand-in-hand with various CI and other automated build systems. `test-patch` will attempt to auto-determine if it is running under such a system and change its defaults to match known configuration parameters automatically. When robots are activated, there is generally some additional/changed behavior:
+
+ * display extra information in the footer
+ * change log entries from file names to URLs
+ * activate `--resetrepo` to keep the directory structure clean
+ * enable the running of unit tests and run them in parallel
+ * if possible, write comments to bug systems
+ * attempt to determine the build tool in use
+ * activate Docker maintenance when `--docker` is passed
+ * attempt to determine whether this is a full build (`qbt`) or testing a patch/merge request/pull request.
+
+Circle CI
+=========
+
+TRIGGER: ${CIRCLECI}=true
+
+Circle CI support in `test-patch` is limited to github.com. Artifacts (the `--patch-dir` directory) location needs to be handled set on the command line. Linking to the logs is not currently supported.
+
+Gitlab CI
+=========
+
+TRIGGER: ${GITLAB_CI}=true
+
+Artifacts, patch logs, etc are configured to go to a yetus-out directory in the source tree after completion. Adding this stanza to your `.gitlab-ci.yml` file will upload and store those components for a week in Gitlab CI's artifact retrieval system:
+
+```yaml
+ artifacts:
+ expire_in: 1 week
+ when: always
+ paths:
+ - yetus-out/
+
+```
+
+Jenkins
+=======
+
+TRIGGER: ${JENKINS_URL}=(anything) , ${EXECUTOR_NUMBER}=(anything)
+
+Jenkins is extremely open-ended and, given multiple executors, does not run workflows in isolation. As a result, many more configuration options generally need to be configured as it is not safe or may be suprising to users for test-patch to autodetermine some settings. By default, Jenkins will trigger a full build.
+
+There is some support for a few well known environment variables:
+ * `${CHANGE_URL}` or `${ghprbPullLink}` will set the patch location as well as trigger some extra handling if 'github' or 'gitlab' appear in the string.
+ * `${GIT_URL}` will trigger the same extra handling if 'github' or 'gitlab' appear in the string.
+ * If `${ghprbPullId}` is set, then test-patch will configure itself for a Github-style PR.
+
+
+See also
+ * See also the source tree's Jenkinsfile for some tips and tricks.
+ * [precommit-admin](precommit-admin), for special utilities built for Jenkins.
+ * [GitHub Branch Source Plugin](https://wiki.jenkins.io/display/JENKINS/GitHub+Branch+Source+Plugin)
+ * [GitHub Pull Request Builder Plugin](https://wiki.jenkins.io/display/JENKINS/GitHub+pull+request+builder+plugin)
+ * https://{your local server}/env-vars.html/
+
+Travis CI
+=========
+
+TRIGGER: ${TRAVIS}=true
+
+Travis CI support will update the local checked out source repository to include references to all branches and tags
+
+If `${ARTIFACTS_PATH}` is configured, then `--patch-dir` is set to the first listed directory path. However, links to the location logs must still be configured manually.
+
+Personalities will override the auto-detected Github repository information. It may be necessary to manually configure it in your .travis.yml file.
+
+
+
+Manual Configuration
+====================
+
+For automated systems that are not directly supported, `--robot` tells `test-patch` that this is an automated system. This will trigger many of the above settings.
+
+
+The `--build-url` option is also useful when running in `--robot` mode so that emails and such
+have a location to look at the output artifacts:
+
+```bash
+$ test-patch --robot --build-url=http://server.example.name:80/${buildnumber}/
+```
+
+Some plug-ins such as Maven have special handling if there are multiple executions of `test-patch` happening at once. It is very common when using automation systems to have multiple runs on the same host. In order to assist these plug-ins, an instance identifier may be provided:
+
+```bash
+$ test-patch --robot --instance=1
+```
+
+If `--robot` is specified without an instance, a random number is generated and used.
+
+
+Sentinel Mode
+=============
+
+If stuck Docker containers are a problem, a more aggressive robot may be enabled with the `--sentinel` option. This option enables killing containers that have been running for over 24 hours as well.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/core.d/01-common.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/core.d/01-common.sh b/precommit/src/main/shell/core.d/01-common.sh
index 0d89106..67d2e46 100755
--- a/precommit/src/main/shell/core.d/01-common.sh
+++ b/precommit/src/main/shell/core.d/01-common.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+
+
## @description Setup the default global variables
## @audience public
## @stability stable
@@ -23,15 +25,19 @@ function common_defaults
#shellcheck disable=SC2034
BASEDIR=$(pwd)
BUGSYSTEMS=""
- BUILDTOOL="nobuild"
+ BUILDTOOL=""
BUILDTOOLS=""
#shellcheck disable=SC2034
EXEC_MODES=""
- #shellcheck disable=SC2034
- JENKINS=false
+ ROBOTTYPE=""
LOAD_SYSTEM_PLUGINS=true
#shellcheck disable=SC2034
OFFLINE=false
+ GIT_ASKPASS=${GIT_ASKPASS:-/bin/true}
+ #shellcheck disable=SC2034
+ GIT_OFFLINE=false
+ #shellcheck disable=SC2034
+ GIT_SHALLOW=false
OSTYPE=$(uname -s)
#shellcheck disable=SC2034
PATCH_BRANCH=""
@@ -138,6 +144,14 @@ function common_args
--git-cmd=*)
GIT=${i#*=}
;;
+ --git-offline)
+ #shellcheck disable=SC2034
+ GIT_OFFLINE=true
+ ;;
+ --git-shallow)
+ #shellcheck disable=SC2034
+ GIT_SHALLOW=true
+ ;;
--grep-cmd=*)
GREP=${i#*=}
;;
@@ -151,6 +165,8 @@ function common_args
--offline)
#shellcheck disable=SC2034
OFFLINE=true
+ #shellcheck disable=SC2034
+ GIT_OFFLINE=true
;;
--patch-cmd=*)
PATCH=${i#*=}
@@ -185,6 +201,8 @@ function common_args
esac
done
+ activate_robots "$@"
+
set_yetus_version
if [[ ${showhelp} == true ]]; then
@@ -506,6 +524,12 @@ function importplugins
fi
done
+ if [[ ${ROBOT} == true ]]; then
+ if declare -f "${ROBOTTYPE}"_set_plugin_defaults >/dev/null; then
+ "${ROBOTTYPE}"_set_plugin_defaults
+ fi
+ fi
+
if declare -f personality_globals > /dev/null; then
personality_globals
fi
@@ -640,4 +664,50 @@ function set_yetus_version
| head -1 \
| ${SED} -e 's|^ *<version>||' -e 's|</version>.*$||' 2>/dev/null)
fi
+}
+
+## @description import and set defaults based upon any auto-detected automation
+## @audience private
+## @stability evolving
+## @replaceable yes
+function activate_robots
+{
+ declare -a files
+ declare i
+
+ if [[ -d "${BINDIR}/robots.d" ]]; then
+ for i in "${BINDIR}"/robots.d/*.sh; do
+ if [[ -f ${i} ]]; then
+ yetus_debug "Importing ${i}"
+ #shellcheck disable=SC1090
+ . "${i}"
+ fi
+ done
+ fi
+}
+
+## @description attempt to guess what the build tool should be
+## @audience public
+## @stability evolving
+## @replaceable no
+function guess_build_tool
+{
+ declare plugin
+ declare filename
+
+ for plugin in ${BUILDTOOLS}; do
+ if [[ "${plugin}" != "nobuild" ]] && declare -f "${plugin}_buildfile" >/dev/null 2>&1; then
+ filename=$("${plugin}_buildfile")
+ if [[ -n "${filename}" ]] &&
+ [[ -f "${BASEDIR}/${filename}" ]]; then
+ BUILDTOOL=${plugin}
+ fi
+ fi
+ done
+
+ if [[ -z ${BUILDTOOL} ]]; then
+ BUILDTOOL=nobuild
+ fi
+
+ echo "Setting build tool to ${BUILDTOOL}"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/core.d/docker.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh
index 83efc3b..a9d0745 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -27,6 +27,10 @@ DOCKER_MEMORY="4g"
declare -a DOCKER_EXTRAARGS
+DOCKER_EXTRAENVS+=("JAVA_HOME")
+DOCKER_EXTRAENVS+=("PATCH_SYSTEM")
+DOCKER_EXTRAENVS+=("PROJECT_NAME")
+
####
#### IMPORTANT
####
@@ -505,6 +509,32 @@ function docker_version
echo "${val}"
}
+## @description Queue env vars to add to the docker env
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param envname
+## @param ...
+function add_docker_env
+{
+ for k in "$@"; do
+ DOCKER_EXTRAENVS+=("${k}")
+ done
+}
+
+## @description Do the work to add the env vars onto the Docker cmd
+## @audience private
+## @stability stable
+## @replaceable yes
+function docker_do_env_adds
+{
+ declare k
+
+ for k in "${DOCKER_EXTRAENVS[@]}"; do
+ DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
+ done
+}
+
## @description Start a test patch docker container
## @audience private
## @stability evolving
@@ -603,12 +633,11 @@ PatchSpecificDocker
DOCKER_EXTRAARGS+=(-w "/testptch/${PROJECT_NAME}")
DOCKER_EXTRAARGS+=("--env=BASEDIR=/testptch/${PROJECT_NAME}")
DOCKER_EXTRAARGS+=("--env=DOCKER_VERSION=${dockerversion} Image:${baseimagename}")
- DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
- DOCKER_EXTRAARGS+=("--env=PATCH_SYSTEM=${PATCH_SYSTEM}")
- DOCKER_EXTRAARGS+=("--env=PROJECT_NAME=${PROJECT_NAME}")
DOCKER_EXTRAARGS+=("--env=TESTPATCHMODE=${TESTPATCHMODE}")
- DOCKER_EXTRAARGS+=(--name "${containername}")
+ docker_do_env_adds
+
+ DOCKER_EXTRAARGS+=(--name "${containername}")
trap 'docker_signal_handler' SIGTERM
trap 'docker_signal_handler' SIGINT
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/core.d/patchfiles.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/core.d/patchfiles.sh b/precommit/src/main/shell/core.d/patchfiles.sh
index 90fb280..9a17876 100755
--- a/precommit/src/main/shell/core.d/patchfiles.sh
+++ b/precommit/src/main/shell/core.d/patchfiles.sh
@@ -114,6 +114,10 @@ function locate_patch
PATCH_SYSTEM=${bugsys}
fi
fi
+ # did the bug system actually make us change our mind?
+ if [[ "${BUILDMODE}" == full ]]; then
+ return 0
+ fi
done
# ok, none of the bug systems know. let's see how smart we are
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/robots.d/circleci.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/robots.d/circleci.sh b/precommit/src/main/shell/robots.d/circleci.sh
new file mode 100755
index 0000000..ec250c9
--- /dev/null
+++ b/precommit/src/main/shell/robots.d/circleci.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no public APIs here
+# SHELLDOC-IGNORE
+
+# shellcheck disable=2034
+if [[ "${CIRCLECI}" = true ]]; then
+ if [[ ${CIRCLE_REPOSITORY_URL} =~ github.com ]]; then
+ # github artifacts show up like so:
+ #BUILD_URL_ARTIFACTS=https://circle-artifacts.com/gh/username/repo/buildnum/artifacts/0/dir/file
+ # but test-patch doesn't support URLs that aren't tied to the build_url. so that
+ # needs to get rewritten first before this can be used
+
+ BUILD_URL="https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${CIRCLE_BUILD_NUM}"
+ BUILD_URL_CONSOLE='/'
+ CONSOLE_USE_BUILD_URL=false
+ ROBOT=true
+ ROBOTTYPE=circleci
+
+
+ yetus_comma_to_array CPR "${CIRCLE_PULL_REQUESTS}"
+
+ if [[ "${#CIRCLE_PULL_REQUESTS[@]}" -ne 1 ]]; then
+ BUILDMODE=full
+ USER_PARAMS+=("--empty-patch")
+ PATCH_BRANCH="${CIRCLE_BRANCH}"
+ pushd "${BASEDIR}" >/dev/null || exit 1
+ "${GIT}" branch --set-upstream-to=origin/"${CIRCLE_BRANCH}" "${CIRCLE_BRANCH}"
+ popd >/dev/null || exit 1
+ else
+ PATCH_OR_ISSUE="${CIRCLE_PULL_REQUEST}"
+ USER_PARAMS+=("${CIRCLE_PULL_REQUEST}")
+ fi
+
+ add_docker_env \
+ CIRCLECI \
+ CIRCLE_BRANCH \
+ CIRCLE_BUILD_NUM \
+ CIRCLE_PULL_REQUEST \
+ CIRCLE_PULL_REQUESTS \
+ CIRCLE_PROJECT_USERNAME \
+ CIRCLE_PROJECT_REPONAME \
+ CIRCLE_REPOSITORY_URL
+
+ yetus_add_entry EXEC_MODES Circle_CI
+ fi
+fi
+
+function circleci_set_plugin_defaults
+{
+ if [[ ${CIRCLE_REPOSITORY_URL} =~ github.com ]]; then
+ if [[ "${#CIRCLE_PULL_REQUESTS[@]}" -eq 1 ]] \
+ && declare -f github_breakup_url >/dev/null 2>&1; then
+ github_breakup_url "${CIRCLE_PULL_REQUEST}"
+ fi
+ GITHUB_REPO=${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}
+ fi
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/robots.d/gitlabci.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/robots.d/gitlabci.sh b/precommit/src/main/shell/robots.d/gitlabci.sh
new file mode 100755
index 0000000..cafcde4
--- /dev/null
+++ b/precommit/src/main/shell/robots.d/gitlabci.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no public APIs here
+# SHELLDOC-IGNORE
+
+# shellcheck disable=2034
+if [[ "${GITLAB_CI}" = true ]]; then
+ CONSOLE_USE_BUILD_URL=true
+ PATCH_DIR=/tmp/yetus-out
+ RELOCATE_PATCH_DIR=true
+ ROBOT=true
+ ROBOTTYPE=gitlabci
+ INSTANCE=${CI_JOB_ID}
+ BUILD_URL=${CI_JOB_URL}
+ BUILD_URL_CONSOLE="/"
+ GITLAB_REPO=${CI_PROJECT_PATH}
+ BUILD_URL_ARTIFACTS=/artifacts/file/yetus-out
+ PATCH_OR_ISSUE="GLSHA:${CI_BUILD_REF}"
+ USER_PARAMS+=("GLSHA:${CI_BUILD_REF}")
+
+ add_docker_env \
+ CI_BUILD_REF \
+ CI_JOB_ID \
+ CI_JOB_URL \
+ CI_PROJECT_PATH \
+ GITLAB_CI
+
+ yetus_add_entry EXEC_MODES Gitlab_CI
+ yetus_add_entry EXEC_MODES ResetRepo
+ yetus_add_entry EXEC_MODES Robot
+ yetus_add_entry EXEC_MODES UnitTests
+fi
+
+function gitlabci_set_plugin_defaults
+{
+ # shellcheck disable=2034
+ GITLAB_REPO=${CI_PROJECT_PATH}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/robots.d/jenkins.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/robots.d/jenkins.sh b/precommit/src/main/shell/robots.d/jenkins.sh
new file mode 100755
index 0000000..b39e95b
--- /dev/null
+++ b/precommit/src/main/shell/robots.d/jenkins.sh
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SHELLDOC-IGNORE
+
+
+# we need two for Jenkins because users may
+# use the jenkins-cli which will also read JENKINS_URL
+# shellcheck disable=SC2034
+if [[ -n "${JENKINS_URL}" && -n "${EXECUTOR_NUMBER}" ]]; then
+ ROBOT=true
+ INSTANCE=${EXECUTOR_NUMBER}
+ ROBOTTYPE=jenkins
+ JENKINS_CUSTOM_HOMEDIR=false
+ BUILDMODE=full
+ USER_PARAMS+=("--empty-patch")
+
+ # if we are running in an Jenkins docker container
+ # spawed by using agent, then there is a good chance
+ # that the user running the test is in a bad state:
+ # no password entry, no home directory, etc.
+ if [[ "${HOME}" == / ]]; then
+ if ! id "(id -u)" >/dev/null 2>&1; then
+ HOME=/tmp/yetus-home.${RANDOM}
+ JENKINS_CUSTOM_HOMEDIR=true
+ yetus_error "WARNING: Invalid HOME env defined by Jenkins, setting to ${HOME}"
+ if ! mkdir -p "${HOME}"; then
+ yetus_error "ERROR: Cannot mkdir ${HOME}. Exiting."
+ exit 1
+ fi
+ fi
+ fi
+
+ git_requires_creds
+
+ if [[ "${GIT_OFFLINE}" == false ]]; then
+ yetus_error "WARNING: Working around Jenkins branch information"
+ pushd "${BASEDIR}" >/dev/null || exit 1
+ "${GIT}" remote set-branches origin '*'
+ "${GIT}" fetch -v
+ popd >/dev/null || exit 1
+ fi
+
+ # BUILD_URL comes from Jenkins already
+ BUILD_URL_CONSOLE=console
+ # shellcheck disable=SC2034
+ CONSOLE_USE_BUILD_URL=true
+
+ # shellcheck disable=SC2034,SC2154
+ if [[ -n "${ghprbPullId}" ]]; then
+ # GitHub Pull Request Builder Plugin
+ PATCH_OR_ISSUE="GH:${ghprbPullId}"
+ fi
+
+ # shellcheck disable=SC2034,SC2154
+ if [[ -n "${ghprbPullLink}" ]]; then
+ # GitHub Pull Request Builder Plugin
+ PATCH_OR_ISSUE="${ghprbPullLink}"
+ fi
+
+ # shellcheck disable=SC2034,SC2154
+ if [[ -n "${CHANGE_URL}" ]]; then
+ # GitHub Branch Source Plugin, among others
+ # likely to be more accurate than ghprbPullId
+ # since it is a full URL
+ PATCH_OR_ISSUE="${CHANGE_URL}"
+ fi
+
+ if [[ -z "${PATCH_OR_ISSUE}" ]] && [[ -n "${BRANCH_NAME}" ]]; then
+ if [[ "${GIT_OFFLINE}" == false ]]; then
+ yetus_error "WARNING: Resetting Jenkins git upstream information"
+ pushd "${BASEDIR}" >/dev/null || exit 1
+ "${GIT}" branch --set-upstream-to=origin/"${BRANCH_NAME}" "${BRANCH_NAME}"
+ popd >/dev/null || exit 1
+ fi
+ PATCH_BRANCH=${BRANCH_NAME}
+ fi
+
+ add_docker_env \
+ BRANCH_NAME \
+ BUILD_URL \
+ CHANGE_URL \
+ EXECUTOR_NUMBER \
+ ghprbPullId \
+ ghprbPullLink \
+ GIT_URL \
+ JENKINS_URL
+
+ yetus_add_entry EXEC_MODES Jenkins
+fi
+
+function jenkins_set_plugin_defaults
+{
+ if [[ -n "${GIT_URL}" ]]; then
+ if [[ "${GIT_URL}" =~ github ]] \
+ && declare -f github_breakup_url >/dev/null 2>&1; then
+ github_breakup_url "${GIT_URL}"
+ elif [[ "${GIT_URL}" =~ gitlab ]] \
+ && declare -f gitlab_breakup_url >/dev/null 2>&1; then
+ gitlab_breakup_url "${GIT_URL}"
+ fi
+ fi
+
+ if [[ -n "${ghprbPullLink}" ]]; then
+ if [[ "${ghprbPullLink}" =~ github ]] \
+ && declare -f github_breakup_url >/dev/null 2>&1; then
+ github_breakup_url "${ghprbPullLink}"
+ fi
+ fi
+
+ if [[ -n "${CHANGE_URL}" ]]; then
+ if [[ "${CHANGE_URL}" =~ github ]] \
+ && declare -f github_breakup_url >/dev/null 2>&1; then
+ github_breakup_url "${CHANGE_URL}"
+ elif [[ "${CHANGE_URL}" =~ gitlab ]] \
+ && declare -f gitlab_breakup_url >/dev/null 2>&1; then
+ gitlab_breakup_url "${CHANGE_URL}"
+ fi
+ fi
+
+ if [[ "${JENKINS_CUSTOM_HOMEDIR}" == true ]]; then
+ # maven defaults to '(cwd)/?/.m2' if user isn't in
+ # /etc/passwd. So force maven custom repos to at least
+ # give us a chance. also need to put this in user params
+ # in case of re-exec
+ USER_PARAMS+=("--mvn-custom-repos")
+ yetus_error "WARNING: Setting --mvn-custom-repos due to previously invalid home directory"
+ MAVEN_CUSTOM_REPOS=true
+ fi
+}
+
+function jenkins_verify_patchdir
+{
+ declare commentfile=$1
+ declare extra
+
+ if [[ -n ${NODE_NAME} ]]; then
+ extra=" (Jenkins node ${NODE_NAME})"
+ fi
+ echo "Jenkins${extra} information at ${BUILD_URL}${BUILD_URL_CONSOLE} may provide some hints. " >> "${commentfile}"
+}
+
+function jenkins_unittest_footer
+{
+ declare statusjdk=$1
+ declare extra
+
+ add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"
+}
+
+function jenkins_finalreport
+{
+ add_footer_table "Console output" "${BUILD_URL}${BUILD_URL_CONSOLE}"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/robots.d/travisci.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/robots.d/travisci.sh b/precommit/src/main/shell/robots.d/travisci.sh
new file mode 100755
index 0000000..9071ffa
--- /dev/null
+++ b/precommit/src/main/shell/robots.d/travisci.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SHELLDOC-IGNORE
+
+if [[ "${TRAVIS}" == true ]]; then
+ # shellcheck disable=SC2034
+ ROBOT=true
+
+ # shellcheck disable=SC2034
+ if [[ -n "${ARTIFACTS_PATH}" ]]; then
+ PATCH_DIR=${ARTIFACTS_PATH%%:*}
+ fi
+
+ # shellcheck disable=SC2034
+ INSTANCE=${TRAVIS_BUILD_ID}
+ # shellcheck disable=SC2034
+ ROBOTTYPE=travisci
+
+ # shellcheck disable=SC2034
+ if [[ "${TRAVIS_PULL_REQUEST}" == false ]]; then
+ BUILDMODE=full
+ PATCH_BRANCH=${TRAVIS_BRANCH}
+ else
+ # shellcheck disable=SC2034
+ BUILDMODE='patch'
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE=GH:${TRAVIS_PULL_REQUEST}
+ USER_PARAMS+=("GH:${TRAVIS_PULL_REQUEST}")
+ fi
+
+ # shellcheck disable=SC2034
+ GITHUB_REPO=${TRAVIS_REPO_SLUG}
+
+ add_docker_env \
+ TRAVIS \
+ TRAVIS_BRANCH \
+ TRAVIS_BUILD_ID \
+ TRAVIS_PULL_REQUEST \
+ TRAVIS_REPO_SLUG
+
+ # shellcheck disable=SC2034
+ BUILD_URL_CONSOLE=console
+ # shellcheck disable=SC2034
+ CONSOLE_USE_BUILD_URL=true
+
+ if [[ -d ${BASEDIR}/.git ]]; then
+ echo "Updating the local git repo to include all branches/tags:"
+ pushd "${BASEDIR}" >/dev/null || exit 1
+ "${GIT}" config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
+ "${GIT}" fetch --tags
+ pushd "${BASEDIR}" >/dev/null || exit 1
+ fi
+
+ yetus_add_entry EXEC_MODES TravisCI
+fi
+
+function travisci_set_plugin_defaults
+{
+ # shellcheck disable=SC2034
+ GITHUB_REPO=${TRAVIS_REPO_SLUG}
+}
+
+#function travisci_verify_patchdir
+#{
+# declare commentfile=$1
+#}
+
+#function travisci_unittest_footer
+#{
+# declare statusjdk=$1
+#}
+
+#function travisci_finalreport
+#{
+# declare foo
+#}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/test-patch-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile
index 81f2458..080e26c 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -36,6 +36,7 @@ ENV DEBCONF_TERSE true
######
# Install some basic Apache Yetus requirements
+# some git repos need ssh-client so do it too
######
RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
curl \
@@ -44,6 +45,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
pkg-config \
rsync \
software-properties-common \
+ ssh-client \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
@@ -65,6 +67,7 @@ ENV LC_ALL en_US.UTF-8
RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
####
# Install ant
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/test-patch.d/github.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/github.sh b/precommit/src/main/shell/test-patch.d/github.sh
index 979c085..ce6fa2f 100755
--- a/precommit/src/main/shell/test-patch.d/github.sh
+++ b/precommit/src/main/shell/test-patch.d/github.sh
@@ -119,6 +119,7 @@ function github_jira_bridge
## @description given a URL, break it up into github plugin globals
## @description this will *override* any personality or yetus defaults
+## @description WARNING: Called from the Jenkins support system!
## @param url
function github_breakup_url
{
@@ -129,20 +130,25 @@ function github_breakup_url
count=${url//[^\/]}
count=${#count}
- ((pos2=count-3))
- ((pos1=pos2))
+ if [[ ${count} -gt 4 ]]; then
+ ((pos2=count-3))
+ ((pos1=pos2))
- GITHUB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/)
+ GITHUB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/)
- ((pos1=pos1+1))
- ((pos2=pos1+1))
+ ((pos1=pos1+1))
+ ((pos2=pos1+1))
- GITHUB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/)
+ GITHUB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/)
- ((pos1=pos2+2))
- unset pos2
+ ((pos1=pos2+2))
+ unset pos2
- GITHUB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.)
+ GITHUB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.)
+ else
+ GITHUB_BASE_URL=$(echo "${url}" | cut -f1-3 -d/)
+ GITHUB_REPO=$(echo "${url}" | cut -f4- -d/)
+ fi
}
@@ -405,12 +411,12 @@ function github_locate_patch
fi
case "${input}" in
- GH:*)
- github_locate_pr_patch "${input}" "${output}"
- ;;
GHSHA:*)
github_locate_sha_patch "${input}" "${output}"
;;
+ *)
+ github_locate_pr_patch "${input}" "${output}"
+ ;;
esac
}
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/test-patch.d/gitlab.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/gitlab.sh b/precommit/src/main/shell/test-patch.d/gitlab.sh
index a3301d7..a86754d 100755
--- a/precommit/src/main/shell/test-patch.d/gitlab.sh
+++ b/precommit/src/main/shell/test-patch.d/gitlab.sh
@@ -28,8 +28,8 @@ GITLAB_BASE_URL="https://gitlab.com"
# API interface URL.
GITLAB_API_URL="https://gitlab.com/api/v4"
-# user/repo
-GITLAB_REPO=""
+# user/repo -- this might get set by robots
+GITLAB_REPO=${GITLAB_REPO:-""}
GITLAB_REPO_ENC=""
# user settings
@@ -78,6 +78,7 @@ function gitlab_initialize
## @description given a URL, break it up into gitlab plugin globals
## @description this will *override* any personality or yetus defaults
+## @description WARNING: Called from the Jenkins support system!
## @param url
function gitlab_breakup_url
{
@@ -88,20 +89,25 @@ function gitlab_breakup_url
count=${url//[^\/]}
count=${#count}
- ((pos2=count-3))
- ((pos1=pos2))
+ if [[ ${count} -gt 4 ]]; then
+ ((pos2=count-3))
+ ((pos1=pos2))
- GITLAB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/)
+ GITLAB_BASE_URL=$(echo "${url}" | cut -f1-${pos2} -d/)
- ((pos1=pos1+1))
- ((pos2=pos1+1))
+ ((pos1=pos1+1))
+ ((pos2=pos1+1))
- GITLAB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/)
+ GITLAB_REPO=$(echo "${url}" | cut -f${pos1}-${pos2} -d/)
- ((pos1=pos2+2))
- unset pos2
+ ((pos1=pos2+2))
+ unset pos2
- GITLAB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.)
+ GITLAB_ISSUE=$(echo "${url}" | cut -f${pos1}-${pos2} -d/ | cut -f1 -d.)
+ else
+ GITLAB_BASE_URL=$(echo "${url}" | cut -f1-3 -d/)
+ GITLAB_REPO=$(echo "${url}" | cut -f4- -d/)
+ fi
}
function gitlab_determine_issue
@@ -261,8 +267,8 @@ function gitlab_locate_sha_patch
-H "${gitlabauth}" \
--output "${PATCH_DIR}/gitlab-search.json" \
--location \
+ --silent \
"${GITLAB_API_URL}/projects/${GITLAB_REPO_ENC}/repository/commits/${GITLAB_COMMITSHA}/merge_requests"; then
- cat "${PATCH_DIR}/gitlab-search.json"
return 1
fi
@@ -270,8 +276,22 @@ function gitlab_locate_sha_patch
mrid=$(cut -f2 -d, "${PATCH_DIR}/gitlab-search.json")
mrid=${mrid/\"iid\":}
- gitlab_locate_mr_patch "GL:${mrid}" "${output}"
+ # this is a dumb hack to work around gitlab-org/gitlab-ce#15280
+ if [[ "${mrid}" = '[]' ]] && [[ "${GITLAB_CI}" = true ]]; then
+
+ echo "This appears to be a full build due to gitlab-org/gitlab-ce#15280. Switching modes."
+ PATCH_BRANCH=${CI_COMMIT_REF_NAME}
+
+ # shellcheck disable=SC2034
+ PATCH_OR_ISSUE=""
+ # shellcheck disable=SC2034
+ BUILDMODE=full
+ set_buildmode
+ return 0
+ fi
+
+ gitlab_locate_mr_patch "GL:${mrid}" "${output}"
}
## @description Handle the various ways to reference a gitlab MR
@@ -293,12 +313,12 @@ function gitlab_locate_patch
fi
case "${input}" in
- GL:*)
- gitlab_locate_mr_patch "${input}" "${output}"
- ;;
GLSHA:*)
gitlab_locate_sha_patch "${input}" "${output}"
;;
+ *)
+ gitlab_locate_mr_patch "${input}" "${output}"
+ ;;
esac
}
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/test-patch.d/maven.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/maven.sh b/precommit/src/main/shell/test-patch.d/maven.sh
index 12fb239..13e2867 100755
--- a/precommit/src/main/shell/test-patch.d/maven.sh
+++ b/precommit/src/main/shell/test-patch.d/maven.sh
@@ -58,7 +58,7 @@ function maven_ws_replace
{
declare previous=${MAVEN_CUSTOM_REPOS_DIR}
- if [[ ${JENKINS} == true ]] && [[ -n "${WORKSPACE}" ]]; then
+ if [[ ${ROBOTTYPE} == jenkins ]] && [[ -n "${WORKSPACE}" ]]; then
MAVEN_CUSTOM_REPOS_DIR=$(echo "${MAVEN_CUSTOM_REPOS_DIR}" | "${SED}" -e "s,@@@WORKSPACE@@@,${WORKSPACE},g" )
else
MAVEN_CUSTOM_REPOS_DIR=$(echo "${MAVEN_CUSTOM_REPOS_DIR}" | "${SED}" -e "s,@@@WORKSPACE@@@,${HOME},g" )
@@ -150,7 +150,10 @@ function maven_initialize
return 1
elif [[ ! -d "${MAVEN_CUSTOM_REPOS_DIR}" ]]; then
yetus_debug "Creating ${MAVEN_CUSTOM_REPOS_DIR}"
- mkdir -p "${MAVEN_CUSTOM_REPOS_DIR}"
+ if ! mkdir -p "${MAVEN_CUSTOM_REPOS_DIR}"; then
+ yetus_error "ERROR: Cannot create ${MAVEN_CUSTOM_REPOS_DIR}"
+ return 1
+ fi
fi
fi
@@ -160,7 +163,11 @@ function maven_initialize
return 1
elif [[ ! -e "${HOME}/.m2" ]]; then
yetus_debug "Creating ${HOME}/.m2"
- mkdir -p "${HOME}/.m2"
+ if ! mkdir -p "${HOME}/.m2"; then
+ yetus_error "ERROR: ${HOME}/.m2 cannot be created. " \
+ "See --mvn-custom-repos and --mvn-custom-repos-dir to set a different location."
+ return 1
+ fi
fi
}
http://git-wip-us.apache.org/repos/asf/yetus/blob/60745a0d/precommit/src/main/shell/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index 7d9c363..5f25afb 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -89,14 +89,13 @@ function setup_defaults
# shellcheck disable=SC2034
CHANGED_UNION_MODULES=""
- GIT_OFFLINE=false
PROC_LIMIT=1000
REEXECED=false
RESETREPO=false
- BUILDMODE=patch
+ BUILDMODE=${BUILDMODE:-patch}
# shellcheck disable=SC2034
- BUILDMODEMSG="The patch"
- ISSUE=""
+ BUILDMODEMSG=${BUILDMODEMSG:-"The patch"}
+ ISSUE=${ISSUE:-""}
TIMER=$("${AWK}" 'BEGIN {srand(); print srand()}')
JVM_REQUIRED=true
yetus_add_entry JDK_TEST_LIST compile
@@ -325,7 +324,7 @@ function prepopulate_footer
add_footer_table "Personality" "${PERSONALITY}"
fi
- gitrev=$(${GIT} rev-parse --verify --short HEAD)
+ gitrev=$("${GIT}" rev-parse --verify --short HEAD)
add_footer_table "git revision" "${PATCH_BRANCH} / ${gitrev}"
}
@@ -473,7 +472,6 @@ function write_comment
function verify_patchdir_still_exists
{
local -r commentfile=/tmp/testpatch.$$.${RANDOM}
- local extra=""
if [[ ! -d ${PATCH_DIR} ]]; then
rm "${commentfile}" 2>/dev/null
@@ -483,11 +481,10 @@ function verify_patchdir_still_exists
echo
cat ${commentfile}
echo
- if [[ ${JENKINS} == true ]]; then
- if [[ -n ${NODE_NAME} ]]; then
- extra=" (Jenkins node ${NODE_NAME})"
+ if [[ ${ROBOT} == true ]]; then
+ if declare -f "${ROBOTTYPE}"_verify_patchdir >/dev/null; then
+ "${ROBOTTYPE}"_verify_patchdir "${commentfile}"
fi
- echo "Jenkins${extra} information at ${BUILD_URL}${BUILD_URL_CONSOLE} may provide some hints. " >> "${commentfile}"
write_comment ${commentfile}
fi
@@ -735,13 +732,15 @@ function yetus_usage
yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
yetus_add_option "--dirty-workspace" "Allow the local git workspace to have uncommitted changes"
yetus_add_option "--empty-patch" "Create a summary of the current source tree"
+ yetus_add_option "--git-offline" "Do not fail if git cannot do certain remote operations"
+ yetus_add_option "--git-shallow" "Repo does not know about other branches or tags"
yetus_add_option "--java-home=<path>" "Set JAVA_HOME (In Docker mode, this should be local to the image)"
yetus_add_option "--linecomments=<bug>" "Only write line comments to this comma delimited list (defaults to bugcomments)"
yetus_add_option "--list-plugins" "List all installed plug-ins and then exit"
yetus_add_option "--multijdkdirs=<paths>" "Comma delimited lists of JDK paths to use for multi-JDK tests"
yetus_add_option "--multijdktests=<list>" "Comma delimited tests to use when multijdkdirs is used. (default: '${jdktlist}')"
yetus_add_option "--modulelist=<list>" "Specify additional modules to test (comma delimited)"
- yetus_add_option "--offline" "Avoid connecting to the Internet"
+ yetus_add_option "--offline" "Avoid connecting to the network"
yetus_add_option "--patch-dir=<dir>" "The directory for working and output files (default '/tmp/test-patch-${PROJECT_NAME}/pid')"
yetus_add_option "--personality=<file>" "The personality file to load"
yetus_add_option "--proclimit=<num>" "Limit on the number of processes (default: ${PROC_LIMIT})"
@@ -784,9 +783,8 @@ function yetus_usage
yetus_add_option "--console-report-file=<file>" "Save the final console-based report to a file in addition to the screen"
yetus_add_option "--console-urls" "Use the build URL instead of path on the console report"
yetus_add_option "--instance=<string>" "Parallel execution identifier string"
- yetus_add_option "--jenkins" "Enable Jenkins-specifc handling (auto: --robot)"
yetus_add_option "--mv-patch-dir" "Move the patch-dir into the basedir during cleanup"
- yetus_add_option "--robot" "Assume this is an automated run"
+ yetus_add_option "--robot" "Assume this is an automated run (default: auto-detect supported CI system)"
yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)"
yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
@@ -874,15 +872,10 @@ function parse_args
;;
--empty-patch)
BUILDMODE=full
- # shellcheck disable=SC2034
- BUILDMODEMSG="The source tree"
;;
--java-home=*)
JAVA_HOME=${i#*=}
;;
- --jenkins)
- JENKINS=true
- ;;
--linecomments=*)
BUGLINECOMMENTS=${i#*=}
BUGLINECOMMENTS=${BUGLINECOMMENTS//,/ }
@@ -955,7 +948,6 @@ function parse_args
;;
--tpinstance=*)
INSTANCE=${i#*=}
- EXECUTOR_NUMBER=${INSTANCE}
;;
--tpperson=*)
REEXECPERSONALITY=${i#*=}
@@ -984,11 +976,7 @@ function parse_args
exit 1
fi
- if [[ ${JENKINS} = true ]]; then
- ROBOT=true
- INSTANCE=${EXECUTOR_NUMBER}
- yetus_add_entry EXEC_MODES Jenkins
- fi
+ set_buildmode
if [[ ${ROBOT} = true ]]; then
# shellcheck disable=SC2034
@@ -1084,6 +1072,32 @@ function parse_args
fi
}
+## @description Switch BUILDMODE. Callers are responsible for setting
+## @description the appropriate vars. Use with caution.
+## @audience private
+## @stability evolving
+## @replaceable no
+function set_buildmode
+{
+ # if both a patch and --empty-patch has been set, a choice needs
+ # to be made. If our exec is called qbt, then go full.
+ # otherwise, defer to the patch
+ if [[ -n "${PATCH_OR_ISSUE}" && "${BUILDMODE}" == full ]]; then
+ if [[ "${BINNAME}" =~ qbt ]]; then
+ BUILDMODE="full"
+ else
+ BUILDMODE="patch"
+ fi
+ fi
+
+ if [[ "${BUILDMODE}" == full ]]; then
+ # shellcheck disable=SC2034
+ BUILDMODEMSG="The source tree"
+ else
+ BUILDMODEMSG="The patch"
+ fi
+}
+
## @description Locate the build file for a given directory
## @audience private
## @stability stable
@@ -1288,7 +1302,8 @@ function find_changed_modules
fi
}
-## @description check if repo requires ssh creds to pull
+## @description check if repo requires creds to do remote operations
+## @description also sets and uses GIT_OFFLINE as appropriate
## @audience private
## @stability stable
## @replaceable no
@@ -1297,16 +1312,20 @@ function find_changed_modules
function git_requires_creds
{
declare status
- declare -a remotes
- # shellcheck disable=SC2207
- remotes=( $("${GIT}" remote -v show -n) )
+ if [[ "${GIT_OFFLINE}" == true ]]; then
+ return 1
+ fi
- for r in "${remotes[@]}"; do
- if [[ ${r} =~ /@/ ]] || [[ ${r} =~ git:// ]]; then
- return 1
- fi
- done
+ pushd "${BASEDIR}" >/dev/null || cleanup_and_exit 1
+
+ if ! "${GIT}" fetch --dry-run >/dev/null 2>&1; then
+ GIT_OFFLINE=true
+ return 1
+ fi
+
+ popd >/dev/null || cleanup_and_exit 1
+ GIT_OFFLINE=false
return 0
}
@@ -1322,28 +1341,25 @@ function git_checkout
declare currentbranch
declare exemptdir
declare status
- declare pullmayfail=false
big_console_header "Confirming git environment"
+ git_requires_creds
+
cd "${BASEDIR}" || cleanup_and_exit 1
if [[ ! -e .git ]]; then
yetus_error "ERROR: ${BASEDIR} is not a git repo."
cleanup_and_exit 1
fi
- if git_requires_creds; then
- pullmayfail=true
- fi
-
if [[ ${RESETREPO} == "true" ]] ; then
if [[ -d .git/rebase-apply ]]; then
yetus_error "ERROR: a previous rebase failed. Aborting it."
- ${GIT} rebase --abort
+ "${GIT}" rebase --abort
fi
- if ! ${GIT} reset --hard; then
+ if ! "${GIT}" reset --hard; then
yetus_error "ERROR: git reset is failing"
cleanup_and_exit 1
fi
@@ -1370,33 +1386,30 @@ function git_checkout
cleanup_and_exit 1
fi
- if ! ${GIT} checkout --force "${PATCH_BRANCH_DEFAULT}"; then
- yetus_error "ERROR: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing"
- cleanup_and_exit 1
+ if [[ "${GIT_SHALLOW}" == false ]]; then
+ if ! "${GIT}" checkout --force "${PATCH_BRANCH_DEFAULT}"; then
+ yetus_error "WARNING: git checkout --force ${PATCH_BRANCH_DEFAULT} is failing; assuming shallow"
+ GIT_SHALLOW=true
+ fi
fi
determine_branch
# we need to explicitly fetch in case the
# git ref hasn't been brought in tree yet
- if [[ ${OFFLINE} == false ]] && [[ ${GIT_OFFLINE} == false ]]; then
-
- if ! ${GIT} pull --rebase; then
- if [[ ${pullmayfail} == true ]]; then
- yetus_error "WARNING: Noted that pull failed, will treat git as offline from here on out"
- GIT_OFFLINE=true
- else
+ if [[ ${GIT_OFFLINE} == false ]]; then
+ if ! "${GIT}" pull --rebase; then
yetus_error "ERROR: git pull is failing"
cleanup_and_exit 1
- fi
fi
fi
- # forcibly checkout this branch or git ref
-
- if ! ${GIT} checkout --force "${PATCH_BRANCH}"; then
- yetus_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
- cleanup_and_exit 1
+ if [[ ${GIT_SHALLOW} == false ]]; then
+ # forcibly checkout this branch or git ref
+ if ! "${GIT}" checkout --force "${PATCH_BRANCH}"; then
+ yetus_error "ERROR: git checkout ${PATCH_BRANCH} is failing"
+ cleanup_and_exit 1
+ fi
fi
# if we've selected a feature branch that has new changes
@@ -2350,8 +2363,10 @@ function check_unittests
modules_messages patch unit false
- if [[ ${JENKINS} == true ]]; then
- add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"
+ if [[ "${ROBOT}" == true ]]; then
+ if declare -f "${ROBOTTYPE}"_unittest_footer >/dev/null; then
+ "${ROBOTTYPE}"_unittest_footer "${statusjdk}"
+ fi
fi
if [[ ${result} -gt 0 ]]; then
@@ -2405,10 +2420,10 @@ function bugsystem_finalreport
declare version
declare bugs
- if [[ "${ROBOT}" = true &&
- -n "${BUILD_URL}" &&
- -n "${BUILD_URL_CONSOLE}" ]]; then
- add_footer_table "Console output" "${BUILD_URL}${BUILD_URL_CONSOLE}"
+ if [[ "${ROBOT}" = true ]]; then
+ if declare -f "${ROBOTTYPE}"_finalreport >/dev/null; then
+ "${ROBOTTYPE}"_finalreport
+ fi
fi
add_footer_table "Powered by" "Apache Yetus ${VERSION} http://yetus.apache.org"
@@ -2428,16 +2443,22 @@ function cleanup_and_exit
{
local result=$1
- if [[ ${ROBOT} == "true" && ${RELOCATE_PATCH_DIR} == "true" && \
- -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then
- # if PATCH_DIR is already inside BASEDIR, then
- # there is no need to move it since we assume that
- # Jenkins or whatever already knows where it is at
- # since it told us to put it there!
- relative_dir "${PATCH_DIR}" >/dev/null
- if [[ $? == 1 ]]; then
- yetus_debug "mv ${PATCH_DIR} ${BASEDIR}"
- mv "${PATCH_DIR}" "${BASEDIR}"
+ if [[ ${ROBOT} == "true" ]]; then
+ if declare -f "${ROBOTTYPE}"_cleanup_and_exit >/dev/null; then
+ "${ROBOTTYPE}"_cleanup_and_exit "${result}"
+ fi
+
+ if [[ ${RELOCATE_PATCH_DIR} == "true" && \
+ -e ${PATCH_DIR} && -d ${PATCH_DIR} ]] ; then
+ # if PATCH_DIR is already inside BASEDIR, then
+ # there is no need to move it since we assume that
+ # Jenkins or whatever already knows where it is at
+ # since it told us to put it there!
+ relative_dir "${PATCH_DIR}" >/dev/null
+ if [[ $? == 1 ]]; then
+ yetus_debug "mv ${PATCH_DIR} ${BASEDIR}"
+ mv "${PATCH_DIR}" "${BASEDIR}"
+ fi
fi
fi
big_console_header "Finished build."
@@ -3164,6 +3185,10 @@ function initialize
importplugins
+ if [[ -z "${BUILDTOOL}" ]]; then
+ guess_build_tool
+ fi
+
parse_args_plugins "$@"
if declare -f personality_parse_args >/dev/null; then
@@ -3195,7 +3220,13 @@ function initialize
if [[ "${BUILDMODE}" = patch ]]; then
locate_patch
+ fi
+
+ # locate_patch might have changed our minds
+ if [[ "${BUILDMODE}" = full ]]; then
+ git_checkout
+ else
# from here on out, we'll be in ${BASEDIR} for cwd
# plugins need to pushd/popd if they change.
git_checkout
@@ -3215,11 +3246,6 @@ function initialize
bugsystem_finalreport 1
cleanup_and_exit 1
fi
-
- else
-
- git_checkout
-
fi
find_changed_files
[2/2] yetus git commit: YETUS-714. Support for yamllint
Posted by aw...@apache.org.
YETUS-714. Support for yamllint
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/492a20e0
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/492a20e0
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/492a20e0
Branch: refs/heads/master
Commit: 492a20e05f3cd6283ac1b821d7068aa3b72268c0
Parents: 60745a0
Author: Allen Wittenauer <aw...@apache.org>
Authored: Sun Oct 21 09:55:29 2018 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Dec 13 08:07:19 2018 -0800
----------------------------------------------------------------------
.hadolint.yaml | 2 +-
.rubocop.yml | 15 ++
asf-site-src/data/versions.yml | 1 +
.../src/main/shell/test-patch-docker/Dockerfile | 4 +-
.../src/main/shell/test-patch.d/yamllint.sh | 164 +++++++++++++++++++
5 files changed, 183 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/492a20e0/.hadolint.yaml
----------------------------------------------------------------------
diff --git a/.hadolint.yaml b/.hadolint.yaml
index 6645586..3cdfb90 100644
--- a/.hadolint.yaml
+++ b/.hadolint.yaml
@@ -12,6 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+---
ignored:
- DL3008
http://git-wip-us.apache.org/repos/asf/yetus/blob/492a20e0/.rubocop.yml
----------------------------------------------------------------------
diff --git a/.rubocop.yml b/.rubocop.yml
index 2fd1917..7858c99 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
Metrics/LineLength:
Max: 100
Metrics/MethodLength:
http://git-wip-us.apache.org/repos/asf/yetus/blob/492a20e0/asf-site-src/data/versions.yml
----------------------------------------------------------------------
diff --git a/asf-site-src/data/versions.yml b/asf-site-src/data/versions.yml
index e0f0d2a..068d1da 100644
--- a/asf-site-src/data/versions.yml
+++ b/asf-site-src/data/versions.yml
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+---
releases:
- '0.6.0'
- '0.7.0'
http://git-wip-us.apache.org/repos/asf/yetus/blob/492a20e0/precommit/src/main/shell/test-patch-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile
index 080e26c..981cb5f 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -144,7 +144,7 @@ RUN pip3 install -v pylint==2.1.1
RUN mv /usr/local/bin/pylint /usr/local/bin/pylint3
######
-# Install python2 and pylint2
+# Install python, pylint2, and yamllint
######
RUN apt-get -q update && apt-get -q install --no-install-recommends -y python \
python2.7 \
@@ -155,7 +155,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y python \
python-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
-RUN pip2 install -v pylint==1.9.2 python-dateutil==2.7.3
+RUN pip2 install -v pylint==1.9.2 python-dateutil==2.7.3 yamllint==1.12.1
RUN mv /usr/local/bin/pylint /usr/local/bin/pylint2
#####
http://git-wip-us.apache.org/repos/asf/yetus/blob/492a20e0/precommit/src/main/shell/test-patch.d/yamllint.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/yamllint.sh b/precommit/src/main/shell/test-patch.d/yamllint.sh
new file mode 100755
index 0000000..b3c4a5b
--- /dev/null
+++ b/precommit/src/main/shell/test-patch.d/yamllint.sh
@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no public APIs here
+# SHELLDOC-IGNORE
+
+add_test_type yamllint
+
+YAMLLINT_TIMER=0
+YAMLLINT=${YAMLLINT:-$(command -v yamllint 2>/dev/null)}
+
+# files that are going to get yamllint'd
+YAMLLINT_CHECKFILES=()
+
+function yamllint_filefilter
+{
+ declare filename=$1
+
+ if [[ ${filename} =~ \.yaml$ ]] ||
+ [[ ${filename} =~ \.yml$ ]]; then
+ add_test yamllint
+ yetus_add_array_element YAMLLINT_CHECKFILES "${filename}"
+ fi
+}
+
+function yamllint_precheck
+{
+ if ! verify_command "yamllint" "${YAMLLINT}"; then
+ add_vote_table 0 yamllint "yamllint was not available."
+ delete_test yamllint
+ fi
+
+
+}
+
+function yamllint_logic
+{
+ declare repostatus=$1
+ declare i
+ declare fn
+ declare output
+
+ pushd "${BASEDIR}" >/dev/null || return 1
+
+ for i in "${YAMLLINT_CHECKFILES[@]}"; do
+ if [[ -f "${i}" ]]; then
+ fn=""
+ while read -r; do
+ if [[ -z "${fn}" ]]; then
+ fn=$REPLY
+ elif [[ -n "${REPLY}" ]]; then
+ # (space)line:col(space)error/warning(space)text
+ output=$(echo "${REPLY}" | awk '{$1=$1":"; $2=$2":"; print $0;}')
+ # fn:line:col:(space)error/warning:(space)text
+ echo "${fn}:${output}" >> "${PATCH_DIR}/${repostatus}-yamllint-result.txt"
+ fi
+ done < <("${YAMLLINT}" "${i}")
+ fi
+ done
+ popd > /dev/null || return 1
+}
+
+function yamllint_preapply
+{
+ if ! verify_needed_test yamllint; then
+ return 0
+ fi
+
+ big_console_header "yamllint plugin: ${PATCH_BRANCH}"
+
+ start_clock
+
+ yamllint_logic branch
+
+ # keep track of how much as elapsed for us already
+ YAMLLINT_TIMER=$(stop_clock)
+ return 0
+}
+
+function yamllint_calcdiffs
+{
+ column_calcdiffs "$@"
+}
+
+function yamllint_postapply
+{
+ declare i
+ declare numPrepatch
+ declare numPostpatch
+ declare diffPostpatch
+ declare fixedpatch
+ declare statstring
+
+ if ! verify_needed_test yamllint; then
+ return 0
+ fi
+
+ big_console_header "yamllint plugin: ${BUILDMODE}"
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${YAMLLINT_TIMER}"
+
+ yamllint_logic patch
+
+ calcdiffs \
+ "${PATCH_DIR}/branch-yamllint-result.txt" \
+ "${PATCH_DIR}/patch-yamllint-result.txt" \
+ yamllint \
+ > "${PATCH_DIR}/diff-patch-yamllint.txt"
+
+ # shellcheck disable=SC2016
+ numPrepatch=$(wc -l "${PATCH_DIR}/branch-yamllint-result.txt" | "${AWK}" '{print $1}')
+
+ # shellcheck disable=SC2016
+ numPostpatch=$(wc -l "${PATCH_DIR}/patch-yamllint-result.txt" | "${AWK}" '{print $1}')
+
+ # shellcheck disable=SC2016
+ diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-yamllint.txt" | "${AWK}" '{print $1}')
+
+
+ ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch))
+
+ statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" )
+
+ if [[ ${diffPostpatch} -gt 0 ]] ; then
+ add_vote_table -1 yamllint "${BUILDMODEMSG} ${statstring}"
+ add_footer_table yamllint "@@BASE@@/diff-patch-yamllint.txt"
+ bugsystem_linecomments "yamllint" "${PATCH_DIR}/diff-patch-yamllint.txt"
+ return 1
+ elif [[ ${fixedpatch} -gt 0 ]]; then
+ add_vote_table +1 yamllint "${BUILDMODEMSG} ${statstring}"
+ return 0
+ fi
+
+ add_vote_table +1 yamllint "There were no new yamllint issues."
+ return 0
+}
+
+function yamllint_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ yamllint_preapply
+ else
+ yamllint_postapply
+ fi
+}