You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2019/01/09 23:39:33 UTC
[yetus] branch master updated (bb1aced -> 1d9f8d6)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git.
from bb1aced YETUS-766. links are broken on in-progress page
new 8f31a4c YETUS-764. test-patch running in a pipeline isn't getting killed
new 4c741d4 YETUS-765. test-patch generating linecomments errors
new 3c58542 YETUS-752. Jenkinsfile: only send mail to dev@ on apache.org (addendum)
new 1d9f8d6 YETUS-686. update homebrew formula with missing dependencies
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Formula/.rubocop.yml | 149 +++++++++++++++++++++
Formula/yetus.rb | 52 +++----
Jenkinsfile | 24 +++-
.../documentation/in-progress/precommit-robots.md | 27 +++-
pom.xml | 2 +
precommit/src/main/shell/core.d/docker.sh | 12 +-
precommit/src/main/shell/test-patch.sh | 12 ++
7 files changed, 243 insertions(+), 35 deletions(-)
create mode 100644 Formula/.rubocop.yml
[yetus] 04/04: YETUS-686. update homebrew formula with missing
dependencies
Posted by aw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 1d9f8d67239434d0405459452ba6ef0d16119300
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Sun Dec 30 22:00:36 2018 -0800
YETUS-686. update homebrew formula with missing dependencies
Signed-off-by: Sean Busbey <bu...@apache.org>
---
Formula/.rubocop.yml | 149 +++++++++++++++++++++++++++++++++++++++++++++++++++
Formula/yetus.rb | 52 +++++++++---------
pom.xml | 2 +
3 files changed, 178 insertions(+), 25 deletions(-)
diff --git a/Formula/.rubocop.yml b/Formula/.rubocop.yml
new file mode 100644
index 0000000..864869e
--- /dev/null
+++ b/Formula/.rubocop.yml
@@ -0,0 +1,149 @@
+# BSD 2-Clause License
+
+# Copyright (c) 2009-present, Homebrew contributors
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Taken from Homebrew @ Dec 3, 2018. Modified to pass yamllint
+#
+---
+# make our hashes consistent
+Layout/AlignHash:
+ EnforcedHashRocketStyle: table
+ EnforcedColonStyle: table
+
+# `system` is a special case and aligns on second argument
+Layout/AlignParameters:
+ Enabled: false
+
+# favour parens-less DSL-style arguments
+Lint/AmbiguousOperator:
+ Enabled: false
+
+# this is a bit less "floaty"
+Layout/CaseIndentation:
+ EnforcedStyle: end
+
+# this is a bit less "floaty"
+Layout/EndAlignment:
+ EnforcedStyleAlignWith: start_of_line
+
+# conflicts with DSL-style path concatenation with `/`
+Layout/SpaceAroundOperators:
+ Enabled: false
+
+# Auto-correct is broken (https://github.com/rubocop-hq/rubocop/issues/6300).
+Layout/EmptyLineAfterGuardClause:
+ Enabled: false
+
+# Auto-correct is broken (https://github.com/rubocop-hq/rubocop/issues/6258)
+# and layout is not configurable
+# (https://github.com/rubocop-hq/rubocop/issues/6254).
+Layout/RescueEnsureAlignment:
+ Enabled: false
+
+# favour parens-less DSL-style arguments
+Lint/AmbiguousBlockAssociation:
+ Enabled: false
+
+# so many of these in formulae and can't be autocorrected
+# TODO: fix these as `ruby -w` complains about them.
+Lint/AmbiguousRegexpLiteral:
+ Enabled: false
+
+# assignment in conditions are useful sometimes
+# TODO: add parentheses for these and remove
+Lint/AssignmentInCondition:
+ Enabled: false
+
+# we output how to use interpolated strings too often
+Lint/InterpolationCheck:
+ Enabled: false
+
+# so many of these in formulae and can't be autocorrected
+Lint/ParenthesesAsGroupedExpression:
+ Enabled: false
+
+# most metrics don't make sense to apply for formulae/taps
+Metrics/AbcSize:
+ Enabled: false
+Metrics/ClassLength:
+ Enabled: false
+Metrics/CyclomaticComplexity:
+ Enabled: false
+Metrics/MethodLength:
+ Enabled: false
+Metrics/ModuleLength:
+ Enabled: false
+Metrics/PerceivedComplexity:
+ Enabled: false
+
+# GitHub diff UI wraps beyond 118 characters (so that's the goal)
+Metrics/LineLength:
+ Max: 170
+ # ignore manpage comments and long single-line strings
+ IgnoredPatterns: ['#: ', ' url "', ' mirror "', ' plist_options :']
+
+# our current conditional style is established
+# TODO: enable this when possible
+Style/ConditionalAssignment:
+ Enabled: false
+
+# most of our APIs are internal so don't require docs
+Style/Documentation:
+ Enabled: false
+
+# we want to add this slowly and manually
+# TODO: add to more files
+Style/FrozenStringLiteralComment:
+ Enabled: false
+
+# so many of these in formulae and can't be autocorrected
+Style/GuardClause:
+ Enabled: false
+
+# depends_on a: :b looks weird in formulae.
+Style/HashSyntax:
+ EnforcedStyle: hash_rockets
+ Exclude:
+ - '**/cmd/*.rb'
+
+# ruby style guide favorite
+Style/StringLiterals:
+ EnforcedStyle: double_quotes
+
+# consistency with above
+Style/StringLiteralsInInterpolation:
+ EnforcedStyle: double_quotes
+
+# make things a bit easier to read
+Style/TernaryParentheses:
+ EnforcedStyle: require_parentheses_when_complex
+
+# messes with existing plist/caveats style
+Style/TrailingBodyOnMethodDefinition:
+ Enabled: false
+
+# a bit confusing to non-Rubyists but useful for longer arrays
+Style/WordArray:
+ MinSize: 4
diff --git a/Formula/yetus.rb b/Formula/yetus.rb
index 0be094d..581adab 100644
--- a/Formula/yetus.rb
+++ b/Formula/yetus.rb
@@ -16,35 +16,37 @@
#
# Homebrew formula to install Apache Yetus
class Yetus < Formula
- desc 'Enable contribution and release processes for software projects'
- homepage 'https://yetus.apache.org/'
- url 'https://www.apache.org/dyn/closer.lua?path=/yetus/0.8.0/yetus-0.8.0-bin.tar.gz'
- sha256 'dea3bcec00c54ff27bcdc3f829749c3ea13bacdcd048792f6aae486ecf2e022e'
+ desc "Enable contribution and release processes for software projects"
+ homepage "https://yetus.apache.org/"
+ url "https://www.apache.org/dyn/closer.lua?path=/yetus/0.8.0/yetus-0.8.0-bin.tar.gz"
+ sha256 "dea3bcec00c54ff27bcdc3f829749c3ea13bacdcd048792f6aae486ecf2e022e"
- option 'with-all', 'Build with all dependencies. Note that some dependencies such as '\
- 'Perl::Critic, Pylint, RuboCop and ruby-lint still need to be installed manually.'
+ option "with-all", "Build with all dependencies. Note that some dependencies such as "\
+ "Perl::Critic, Pylint, RuboCop and ruby-lint still need to be installed manually."
dependencies = [
# programming languages
:java,
- 'scala',
+ "scala",
# build tools
- 'ant',
- 'autoconf',
- 'automake',
- 'cmake',
- 'libtool',
- 'gradle',
- 'maven',
+ "ant",
+ "autoconf",
+ "automake",
+ "cmake",
+ "libtool",
+ "gradle",
+ "maven",
# test tools
- 'findbugs',
- 'shellcheck'
+ "hadolint",
+ "shellcheck",
+ "spotbugs",
+ "yamllint"
]
dependencies.each do |dependency|
- if build.with?('all')
+ if build.with?("all")
depends_on dependency
else
depends_on dependency => :optional
@@ -52,17 +54,17 @@ class Yetus < Formula
end
def install
- rm Dir['bin/*.{bat,cmd,dll,exe}']
- inreplace Dir['bin/*'], '$(dirname -- "${BASH_SOURCE-0}")/..', libexec
- libexec.install Dir['*']
+ rm Dir["bin/*.{bat,cmd,dll,exe}"]
+ inreplace Dir["bin/*"], '$(dirname -- "${BASH_SOURCE-0}")/..', libexec
+ libexec.install Dir["*"]
bin.install_symlink Dir["#{libexec}/bin/*"]
end
test do
- system "#{bin}/qbt", '--version'
- system "#{bin}/releasedocmaker", '-V'
- system "#{bin}/shelldocs", '-V'
- system "#{bin}/smart-apply-patch", '--version'
- system "#{bin}/test-patch", '--version'
+ system "#{bin}/qbt", "--version"
+ system "#{bin}/releasedocmaker", "-V"
+ system "#{bin}/shelldocs", "-V"
+ system "#{bin}/smart-apply-patch", "--version"
+ system "#{bin}/test-patch", "--version"
end
end
diff --git a/pom.xml b/pom.xml
index 68e28a7..2d79cc6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -212,11 +212,13 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
+ <version>0.13</version>
<configuration>
<excludeSubprojects>false</excludeSubprojects>
<excludes>
<exclude>.pylintrc</exclude>
<exclude>.rubocop.yml</exclude>
+ <exclude>Formula/.rubocop.yml</exclude>
</excludes>
</configuration>
</plugin>
[yetus] 03/04: YETUS-752. Jenkinsfile: only send mail to dev@ on
apache.org (addendum)
Posted by aw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 3c585421b6738cb8dee2149a832682a0c37f2c48
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Thu Jan 3 17:49:55 2019 -0800
YETUS-752. Jenkinsfile: only send mail to dev@ on apache.org (addendum)
Signed-off-by: Sean Busbey <bu...@apache.org>
---
Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index aee1009..e067994 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -208,7 +208,7 @@ pipeline {
reportFiles: 'report.html',
reportName: 'Yetus Report'
])
- if (env.BRANCH_NAME == 'master') and (env.BUILD_URL.contains('apache.org')) {
+ if ((env.BRANCH_NAME == 'master') && (env.BUILD_URL.contains('apache.org'))) {
emailext(subject: '$DEFAULT_SUBJECT',
body:
'''For more details, see ${BUILD_URL}
[yetus] 02/04: YETUS-765. test-patch generating linecomments errors
Posted by aw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 4c741d411a28dae9e021b1456ea989fc5f1222c0
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Tue Jan 8 06:00:14 2019 -0800
YETUS-765. test-patch generating linecomments errors
Signed-off-by: Sean Busbey <bu...@apache.org>
---
precommit/src/main/shell/test-patch.sh | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index b361289..52a5579 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -2376,6 +2376,10 @@ function bugsystem_linecomments_trigger
return
fi
+ if [[ ! -f "${PATCH_DIR}/linecomments-in.txt" ]]; then
+ return 0
+ fi
+
# sort the file such that all files and lines are now next to each other
sort "${PATCH_DIR}/linecomments-in.txt" > "${PATCH_DIR}/linecomments-sorted.txt"
[yetus] 01/04: YETUS-764. test-patch running in a pipeline isn't
getting killed
Posted by aw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 8f31a4c8c88a314e7c76fa212a9abb75a1c98c94
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Sun Jan 6 10:42:49 2019 -0800
YETUS-764. test-patch running in a pipeline isn't getting killed
Signed-off-by: Sean Busbey <bu...@apache.org>
---
Jenkinsfile | 22 ++++++++++++++++--
.../documentation/in-progress/precommit-robots.md | 27 ++++++++++++++++++++--
precommit/src/main/shell/core.d/docker.sh | 12 ++++++----
precommit/src/main/shell/test-patch.sh | 8 +++++++
4 files changed, 60 insertions(+), 9 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 78a623a..aee1009 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -254,9 +254,27 @@ ${FILE,path="out/brief.txt"}
}
}
- // Jenkins pipeline jobs fill slaves on PRs without this :(
+ // Jenkins has issues in some configurations sending
+ // signals to child processes. Additionally, Github Branch
+ // Source plug-in will quickly fill Jenkins build hosts
+ // with PR directories. This cleanup stanze kills
+ // any left over processes/containers and frees disk space
+ // on exit
cleanup() {
- deleteDir()
+ script {
+ sh '''
+ if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
+ echo "test-patch process appears to still be running: killing"
+ kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
+ sleep 10
+ fi
+ if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
+ echo "test-patch container appears to still be running: killing"
+ docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
+ fi
+ '''
+ deleteDir()
+ }
}
}
}
diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.md b/asf-site-src/source/documentation/in-progress/precommit-robots.md
index faa487c..9c226ac 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-robots.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-robots.md
@@ -113,12 +113,35 @@ pipeline {
JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
}
- ...
-
}
```
+ Experience has shown that certain Jenkins + Java + OS combinations have problems sending signals to child processes. In the case of Apache Yetus, this may result in aborted or workflows that timeout not being properly killed. `test-patch` will write two files in the patch directory that may be helpful to combat this situation if it applies to your particular configuration. `pidfile.txt` contains the master `test-patch` process id and `cidfile.txt` contains the docker container id. T [...]
+
+ ```groovy
+ post {
+ cleanup() {
+ script {
+ sh '''
+ if [ -f "${env.PATCH_DIR}/pidfile.txt" ]; then
+ kill `cat "${env.PATCH_DIR}/pidfile.txt"` || true
+ sleep 5
+ fi
+ if [ -f "${env.PATCH_DIR}/cidfile.txt" ]; then
+ docker kill `cat "${env.PATCH_DIR}/cidfile.txt"` || true
+ sleep 5
+ fi
+ '''
+ ...
+ deletedir()
+ }
+ }
+ }
+ ```
+
+
+
See also
* See also the source tree's `Jenkinsfile` for some tips and tricks.
* [precommit-admin](precommit-admin), for special utilities built for Jenkins.
diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh
index d419bc0..d298d19 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -717,7 +717,7 @@ function docker_run_image
# make the kernel prefer to kill us if we run out of RAM
DOCKER_EXTRAARGS+=("--oom-score-adj" "500")
- DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile")
+ DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile.txt")
if [[ "${DOCKER_IN_DOCKER}" == true ]]; then
if [[ -e "${DOCKER_SOCKET}" ]]; then
@@ -735,8 +735,7 @@ function docker_run_image
DOCKER_EXTRAARGS+=(--name "${containername}")
- trap 'docker_signal_handler' SIGTERM
- trap 'docker_signal_handler' SIGINT
+ trap 'docker_signal_handler' SIGTERM SIGINT SIGHUP
if [[ ${PATCH_DIR} =~ ^/ ]]; then
dockercmd run --rm=true -i \
@@ -758,6 +757,7 @@ function docker_run_image
printf '\n\n'
echo "Cleaning up docker image used for testing."
dockercmd rmi "${patchimagename}" > /dev/null
+ rm "${PATCH_DIR}/cidfile.txt"
cleanup_and_exit ${retval}
}
@@ -769,10 +769,12 @@ function docker_signal_handler
{
declare cid
- cid=$(cat "${PATCH_DIR}/cidfile")
+ cid=$(cat "${PATCH_DIR}/cidfile.txt")
yetus_error "ERROR: Caught signal. Killing docker container:"
- dockercmd kill "${cid}"
+ echo "ERROR: Caught signal. Killing docker container: ${cid}" > "${PATCH_DIR}/signal.log"
+ dockercmd kill "${cid}" | tee -a "${PATCH_DIR}/signal.log"
+ rm "${PATCH_DIR}/cidfile.txt"
yetus_error "ERROR: Exiting."
cleanup_and_exit 143 # 128 + 15 -- SIGTERM
}
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index 4ea5cb1..b361289 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -2470,6 +2470,10 @@ function cleanup_and_exit
big_console_header "Finished build."
fi
+ if [[ "${DOCKERMODE}" != true ]]; then
+ rm "${PATCH_DIR}/pidfile.txt"
+ fi
+
# shellcheck disable=SC2086
exit ${result}
}
@@ -3210,6 +3214,10 @@ function initialize
docker_initialize
fi
+ if [[ "${DOCKERMODE}" != true ]]; then
+ echo "$$" > "${PATCH_DIR}/pidfile.txt"
+ fi
+
plugins_initialize
if [[ ${RESULT} != 0 ]]; then
cleanup_and_exit 1