You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@yetus.apache.org by aw...@apache.org on 2020/10/15 23:59:58 UTC
[yetus] branch main updated: YETUS-992. Create and publish a github
action (more) (#160)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 1a8dbeb YETUS-992. Create and publish a github action (more) (#160)
1a8dbeb is described below
commit 1a8dbebcf07420098f6f1c4fae78f7e19a1fbc28
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Thu Oct 15 16:59:48 2020 -0700
YETUS-992. Create and publish a github action (more) (#160)
* use the github action to test the main tree in addition
to the normal testing
* speed up the github action bits in use by caching maven
and fix some related issues with using a pre-built cache
* add documentation about --ignore-unknown-options
* add --report-unknown-options support and docs
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
.github/workflows/{yetus.yml => action-test.yml} | 37 +++++++++----------
.github/workflows/linkcheck.yml | 12 ++++++-
.github/workflows/yetus.yml | 25 +++++++------
asf-site-src/Dockerfile | 7 +++-
.../precommit/robots/githubactions.html.md | 3 ++
.../in-progress/precommit/usage-intro.html.md | 22 ++++++++++++
precommit/src/main/shell/core.d/01-common.sh | 5 +++
precommit/src/main/shell/test-patch.sh | 23 ++++++------
start-build-env.sh | 42 +++++++++++++++++-----
9 files changed, 124 insertions(+), 52 deletions(-)
diff --git a/.github/workflows/yetus.yml b/.github/workflows/action-test.yml
similarity index 54%
copy from .github/workflows/yetus.yml
copy to .github/workflows/action-test.yml
index 1a5628f..eadb97e 100644
--- a/.github/workflows/yetus.yml
+++ b/.github/workflows/action-test.yml
@@ -14,7 +14,7 @@
# limitations under the License.
---
-name: Apache Yetus
+name: Apache Yetus (Action Test)
on: [push, pull_request] # yamllint disable-line rule:truthy
@@ -29,29 +29,24 @@ jobs:
with:
path: src
fetch-depth: 0
+ - name: maven cache
+ uses: actions/cache@v2
+ with:
+ path: ~/.m2
+ key: yetus-m2-${{ hashFiles('**/pom.xml') }}
- name: test-patch
- run: >-
- "${GITHUB_WORKSPACE}/src/precommit/src/main/shell/test-patch.sh"
- --basedir="${GITHUB_WORKSPACE}/src"
- --patch-dir="${GITHUB_WORKSPACE}/out"
- --docker
- --project=yetus
- --pylint=pylint2
- --java-home=/usr/lib/jvm/java-8-openjdk-amd64
- --brief-report-file="${GITHUB_WORKSPACE}/out/brief.txt"
- --console-report-file="${GITHUB_WORKSPACE}/out/console.txt"
- --html-report-file="${GITHUB_WORKSPACE}/out/report.html"
- --junit-report-xml="${GITHUB_WORKSPACE}/out/junit-report.xml"
- --proclimit=2000
- --bugcomments=console,briefreport,github,htmlout,junit
- --archive-list="checkstyle-errors.xml,spotbugsXml.xml"
- --plugins=all
- --tests-filter=checkstyle,javadoc,rubocop,test4tests
- --docker-cache-from=apache/yetus-base:main
- --github-token="${{ secrets.GITHUB_TOKEN }}"
+ uses: apache/yetus-test-patch-action@main
+ with:
+ basedir: ./src
+ patchdir: ./out
+ buildtool: maven
+ githubtoken: ${{ secrets.GITHUB_TOKEN }}
+ pylint: pylint2
+ pip: pip2
+ testsfilter: checkstyle,javadoc
- name: Artifact output
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
- name: apacheyetuspatchdir
+ name: apacheyetustestpatchactionout
path: ${{ github.workspace }}/out
diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml
index 035aa63..0e3d7fb 100644
--- a/.github/workflows/linkcheck.yml
+++ b/.github/workflows/linkcheck.yml
@@ -18,11 +18,21 @@ name: linkcheck
on: [push, pull_request] # yamllint disable-line rule:truthy
+
+#
+# NOTE: these paths are referenced in start-build-env.sh
+#
+
jobs:
linkcheck:
runs-on: ubuntu-latest
steps:
- - name: checkout repo
+ - name: checkout
uses: actions/checkout@v2
+ - name: maven cache
+ uses: actions/cache@v2
+ with:
+ path: ~/.m2
+ key: yetus-m2-${{ hashFiles('**/pom.xml') }}
- name: test
run: ./start-build-env.sh ./website-tester.sh
diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml
index 1a5628f..8a6fbea 100644
--- a/.github/workflows/yetus.yml
+++ b/.github/workflows/yetus.yml
@@ -29,26 +29,31 @@ jobs:
with:
path: src
fetch-depth: 0
+ - name: maven cache
+ uses: actions/cache@v2
+ with:
+ path: ~/.m2
+ key: yetus-m2-${{ hashFiles('**/pom.xml') }}
- name: test-patch
run: >-
"${GITHUB_WORKSPACE}/src/precommit/src/main/shell/test-patch.sh"
+ --archive-list="checkstyle-errors.xml,spotbugsXml.xml"
--basedir="${GITHUB_WORKSPACE}/src"
- --patch-dir="${GITHUB_WORKSPACE}/out"
- --docker
- --project=yetus
- --pylint=pylint2
- --java-home=/usr/lib/jvm/java-8-openjdk-amd64
--brief-report-file="${GITHUB_WORKSPACE}/out/brief.txt"
+ --bugcomments=console,briefreport,github,htmlout,junit
--console-report-file="${GITHUB_WORKSPACE}/out/console.txt"
+ --docker
+ --docker-cache-from=apache/yetus-base:main
+ --github-token="${{ secrets.GITHUB_TOKEN }}"
--html-report-file="${GITHUB_WORKSPACE}/out/report.html"
+ --java-home=/usr/lib/jvm/java-8-openjdk-amd64
--junit-report-xml="${GITHUB_WORKSPACE}/out/junit-report.xml"
- --proclimit=2000
- --bugcomments=console,briefreport,github,htmlout,junit
- --archive-list="checkstyle-errors.xml,spotbugsXml.xml"
+ --patch-dir="${GITHUB_WORKSPACE}/out"
--plugins=all
+ --proclimit=2000
+ --project=yetus
+ --pylint=pylint2
--tests-filter=checkstyle,javadoc,rubocop,test4tests
- --docker-cache-from=apache/yetus-base:main
- --github-token="${{ secrets.GITHUB_TOKEN }}"
- name: Artifact output
if: ${{ always() }}
uses: actions/upload-artifact@v2
diff --git a/asf-site-src/Dockerfile b/asf-site-src/Dockerfile
index d9c5788..70aaee8 100644
--- a/asf-site-src/Dockerfile
+++ b/asf-site-src/Dockerfile
@@ -31,6 +31,12 @@ RUN apt-get -q update \
apache2 \
pinentry-curses \
pinentry-tty \
+ ruby-eventmachine \
+ ruby-fastimage \
+ ruby-ffi \
+ ruby-listen \
+ ruby-nokogiri \
+ ruby-sassc \
subversion \
sudo \
vim \
@@ -78,7 +84,6 @@ RUN gem install bundler \
middleman-livereload \
middleman-syntax \
nokogiri:1.10.10 \
- rake:13.0.1 \
sassc:2.4.0 \
therubyracer \
tzinfo-data
diff --git a/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md b/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md
index c7e3977..d029c30 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md
@@ -84,6 +84,7 @@ Options currently supported are:
| pylint | same as `--pylint` | pylint3 | [pylint plug-in](../../plugins/pylint) |
| reapermode | same as `--reapermode` | kill | [Advanced Usage](../../advanced) |
| reviveconfig | same as `--revive-config` | `.revive.toml` | [revive plug-in](../../plugins/revive) |
+| testsfilter | same as `--tests-filter` | '' | [Usage Introduction](../../usage-intro) |
Items marked NONE *MUST* be provided in the workflow yaml file.
@@ -94,8 +95,10 @@ Some options are hard-coded to make `test-patch` easier to use:
| `--brief-report-file` | patchdir/brief.txt | [briefreport plug-in](../../plugins/briefreport) |
| `--console-report-file` | patchdir/console.txt | [QBT](../../qbt) |
| `--html-report-file` | patchdir/report.html | [htmlout plug-in](../../plugins/htmlout) |
+| `--ignore-unknown-options` | true | [Usage Introduction](../../usage-intro) |
| `--junit-report-xml` | patchdir/junit-report.xml | [junit plug-in](../../plugins/junit-bugsystem) |
| `--pylint-requirements` | true | [pylint plug-in](../../plugins/pylint) |
+| `--report-unknown-options` | true | [Usage Introduction](../../usage-intro) |
## Manual Configuration
diff --git a/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md b/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md
index 6fb48c0..f9336b3 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md
@@ -36,12 +36,14 @@
* [GITLAB](#gitlab)
* [Generic URLs](#generic-urls)
* [Excluding Files](#excluding-files)
+* ["Warn-only" Test Results](#warn-only-test-results)
* [Project-specific Capabilities](#project-specific-capabilities)
* [Direct Method](#direct-method)
* [Project Method](#project-method)
* [Fork Bomb Protection](#fork-bomb-protection)
* [MultiJDK](#multijdk)
* [Docker](#docker)
+* [Upgrading](#upgrading)
* [In Closing](#in-closing)
<!-- /MarkdownTOC -->
@@ -386,6 +388,18 @@ $ test-patch --plugins=all --excludes=(filename) (other options)
NOTE: for best results, avoid using ^ and $ in your regular expressions. `test-patch` will automatically
anchor the contents of that file.
+# "Warn-only" Test Results
+
+In some cases, test plug-ins may always generate fail results in ways that cannot be avoided or are so
+monumental to fix that you just want to keep track of the results without ever failing the run. `test-patch`
+provides the `--tests-filter` option to do just that; print the results but don't fail the job. For example:
+
+```bash
+$ test-patch --plugins=all --tests-filter=checkstyle,javadoc (other options)
+```
+
+... will always force the `checkstyle` and `javadoc` tests to never vote -1.
+
# Project-specific Capabilities
Due to the extensible nature of the system, `test-patch` allows for projects to define project-specific rules which we call personalities. (How to build those rules is covered elsewhere.) There are two ways to specify which personality to use:
@@ -440,6 +454,14 @@ $ test-patch --docker
This command will do some preliminary setup and then re-execute itself inside a Docker container. For more information on how to provide a custom Dockerfile and other Docker-specific features, see the specific [precommit Docker support](../docker) page and the [Apache Yetus Docker Hub Images](../../../../yetus-docker-image) page for more information on the convenience Docker images.
+# Upgrading
+
+Currently, Apache Yetus is still undergoing incompatible changes from time to time. Despite that, in many cases
+the upgrade process for `test-patch` and friends is usually just verifying what flags are being passed. To help out,
+there is an option to `--ignore-unknown-options` so that `test-patch` does not error out if it is given flags it no
+longer understands. It will print a list of those unknown options in the end report. In situations where that is
+also undesirable, the `--report-unknown-options` may also be set simultaneously to remove the list from the report.
+
# In Closing
`test-patch` has many other features and command line options for the basic user. Many of these are self-explanatory. To see the list of options, run `test-patch` without any options or with `--help`.
diff --git a/precommit/src/main/shell/core.d/01-common.sh b/precommit/src/main/shell/core.d/01-common.sh
index 9e7d897..0f37733 100755
--- a/precommit/src/main/shell/core.d/01-common.sh
+++ b/precommit/src/main/shell/core.d/01-common.sh
@@ -212,6 +212,11 @@ function common_args
delete_parameter "${i}"
PROJECT_NAME=${i#*=}
;;
+ --report-unknown-options=*)
+ delete_parameter "${i}"
+ #shellcheck disable=SC2034
+ REPORT_UNKNOWN_OPTIONS=${i#*=}
+ ;;
--rsync-cmd=*)
delete_parameter "${i}"
RSYNC=${i#*=}
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index a48c3cb..63ed1dc 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -96,6 +96,7 @@ function setup_defaults
PROC_LIMIT=1000
REEXECED=false
RESETREPO=false
+ REPORT_UNKNOWN_OPTIONS=true
BUILDMODE=${BUILDMODE:-patch}
# shellcheck disable=SC2034
BUILDMODEMSG=${BUILDMODEMSG:-"The patch"}
@@ -638,14 +639,13 @@ function yetus_usage
echo ""
yetus_add_option "--archive-list=<list>" "Comma delimited list of pattern matching notations to copy to patch-dir"
yetus_add_option "--basedir=<dir>" "The directory to apply the patch to (default: current directory)"
- yetus_add_option "--branch=<ref>" "Forcibly set the branch"
yetus_add_option "--branch-default=<ref>" "If the branch isn't forced and we don't detect one in the patch name, use this branch (default 'main')"
+ yetus_add_option "--branch=<ref>" "Forcibly set the branch"
+ yetus_add_option "--bugcomments=<bug>" "Only write comments to the screen and this comma delimited list (default: '${bugsys}')"
yetus_add_option "--build-native=<bool>" "If true, then build native components (default 'true')"
- # shellcheck disable=SC2153
yetus_add_option "--build-tool=<tool>" "Pick which build tool to focus around (default: autodetect from '${buildtools}')"
- yetus_add_option "--bugcomments=<bug>" "Only write comments to the screen and this comma delimited list (default: '${bugsys}')"
- yetus_add_option "--contrib-guide=<url>" "URL to point new users towards project conventions. (default: ${PATCH_NAMING_RULE} )"
yetus_add_option "--continuous-improvement=<bool>" "If true, then do not exit with failure on branches (default: ${CONTINUOUS_IMPROVEMENT})"
+ yetus_add_option "--contrib-guide=<url>" "URL to point new users towards project conventions. (default: ${PATCH_NAMING_RULE} )"
yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
yetus_add_option "--dirty-workspace" "Allow the local git workspace to have uncommitted changes"
yetus_add_option "--empty-patch" "Create a summary of the current source tree"
@@ -656,15 +656,16 @@ function yetus_usage
yetus_add_option "--java-home=<path>" "Set JAVA_HOME (In Docker mode, this should be local to the image)"
yetus_add_option "--linecomments=<bug>" "Only write line comments to this comma delimited list (default: same as --bugcomments)"
yetus_add_option "--list-plugins" "List all installed plug-ins and then exit"
+ yetus_add_option "--modulelist=<list>" "Specify additional modules to test (comma delimited)"
yetus_add_option "--multijdkdirs=<paths>" "Comma delimited lists of JDK paths to use for multi-JDK tests"
yetus_add_option "--multijdktests=<list>" "Comma delimited tests to use when multijdkdirs is used. (default: '${jdktlist}')"
- yetus_add_option "--modulelist=<list>" "Specify additional modules to test (comma delimited)"
yetus_add_option "--offline" "Avoid connecting to the network"
yetus_add_option "--patch-dir=<dir>" "The directory for working and output files (default '/tmp/test-patch-${PROJECT_NAME}/pid')"
yetus_add_option "--personality=<file>" "The personality file to load"
+ yetus_add_option "--plugins=<list>" "Specify which plug-ins to add/delete (comma delimited; use 'all' for all found) e.g. --plugins=all,-ant,-scalac (all plugins except ant and scalac)"
yetus_add_option "--proclimit=<num>" "Limit on the number of processes (default: ${PROC_LIMIT})"
yetus_add_option "--project=<name>" "The short name for project currently using test-patch (default 'yetus')"
- yetus_add_option "--plugins=<list>" "Specify which plug-ins to add/delete (comma delimited; use 'all' for all found) e.g. --plugins=all,-ant,-scalac (all plugins except ant and scalac)"
+ yetus_add_option "--report-unknown-options=<bool>" "Print a warning in the report if --ignore-unknown-options=true and unknown options were found (default: ${REPORT_UNKNOWN_OPTIONS})"
yetus_add_option "--resetrepo" "Forcibly clean the repo"
yetus_add_option "--run-tests" "Run all relevant tests below the base directory"
yetus_add_option "--skip-dirs=<list>" "Skip following directories for module finding"
@@ -672,8 +673,8 @@ function yetus_usage
yetus_add_option "--summarize=<bool>" "Allow tests to summarize results"
yetus_add_option "--test-parallel=<bool>" "Run multiple tests in parallel (default false in developer mode, true in Jenkins mode)"
yetus_add_option "--test-threads=<int>" "Number of tests to run in parallel (default defined in ${PROJECT_NAME} build)"
- yetus_add_option "--unit-test-filter-file=<file>" "The unit test filter file to load"
yetus_add_option "--tests-filter=<list>" "Lists of tests to turn failures into warnings"
+ yetus_add_option "--unit-test-filter-file=<file>" "The unit test filter file to load"
yetus_add_option "--user-plugins=<dir>" "A directory of user provided plugins. see test-patch.d for examples (default empty)"
yetus_add_option "--version" "Print release version information and exit"
@@ -696,9 +697,9 @@ function yetus_usage
echo ""
echo "Automation options:"
- yetus_add_option "--build-url=<url>" "Set the build location web page (Default: '${BUILD_URL}')"
- yetus_add_option "--build-url-console=<location>" "Location relative to --build-url of the console (Default: '${BUILD_URL_CONSOLE}')"
yetus_add_option "--build-url-artifacts=<location>" "Location relative to --build-url of the --patch-dir (Default: '${BUILD_URL_ARTIFACTS}')"
+ yetus_add_option "--build-url-console=<location>" "Location relative to --build-url of the console (Default: '${BUILD_URL_CONSOLE}')"
+ yetus_add_option "--build-url=<url>" "Set the build location web page (Default: '${BUILD_URL}')"
yetus_add_option "--console-report-file=<file>" "Save the final console-based report to a file in addition to the screen"
yetus_add_option "--console-urls" "Use the build URL instead of path on the console report"
yetus_add_option "--instance=<string>" "Parallel execution identifier string"
@@ -3248,7 +3249,9 @@ function initialize
if [[ "${#PARAMETER_TRACKER}" -gt 0 ]]; then
yetus_error "ERROR: Unprocessed flag(s): ${PARAMETER_TRACKER[*]}"
if [[ "${IGNORE_UNKNOWN_OPTIONS}" == true ]]; then
- add_vote_table_v2 "-0" yetus "" "Unprocessed flag(s): ${PARAMETER_TRACKER[*]}"
+ if [[ "${REPORT_UNKNOWN_OPTIONS}" == true ]]; then
+ add_vote_table_v2 "-0" yetus "" "Unprocessed flag(s): ${PARAMETER_TRACKER[*]}"
+ fi
else
add_vote_table_v2 -1 yetus "" "Unprocessed flag(s): ${PARAMETER_TRACKER[*]}"
bugsystem_finalreport 1
diff --git a/start-build-env.sh b/start-build-env.sh
index 1b8af64..aeb1bff 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -15,10 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-if [[ "${GITHUB_ACTIONS}" == true ]]; then
- echo "::group::start-build-env"
-fi
-
set -e # exit on error
ROOTDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")" >/dev/null && pwd -P)
@@ -33,11 +29,20 @@ if [[ "${BRANCH}" =~ HEAD ]]; then
fi
BRANCH=${BRANCH//\//_}
+if [[ "${GITHUB_ACTIONS}" == true ]]; then
+ echo "::group::start-build-env - warm docker cache"
+fi
+
echo "Attempting a few pulls of ${YETUS_DOCKER_REPO} and ${YETUS_DOCKER_REPO}-base to save time"
echo "Errors here will be ignored!"
docker pull "${YETUS_DOCKER_REPO}-base:${BRANCH}" || docker pull "${YETUS_DOCKER_REPO}-base:main" || true
docker pull "${YETUS_DOCKER_REPO}:${BRANCH}" || docker pull "${YETUS_DOCKER_REPO}:main" || true
+if [[ "${GITHUB_ACTIONS}" == true ]]; then
+ echo "::endgroup::"
+ echo "::group::start-build-env - rebuild base"
+fi
+
docker build \
--cache-from="${YETUS_DOCKER_REPO}-build:${BRANCH},${YETUS_DOCKER_REPO}-base:${BRANCH},${YETUS_DOCKER_REPO}-base:main,${YETUS_DOCKER_REPO}:${BRANCH},${YETUS_DOCKER_REPO}:main" \
-t "${YETUS_DOCKER_REPO}-build:${BRANCH}" .
@@ -72,6 +77,11 @@ if command -v selinuxenabled >/dev/null && selinuxenabled; then
fi
fi
+if [[ "${GITHUB_ACTIONS}" == true ]]; then
+ echo "::endgroup::"
+ echo "::group::start-build-env - build asf-site-src container"
+fi
+
cd "${ROOTDIR}/asf-site-src"
docker build \
-t "${YETUS_DOCKER_REPO}-build-${USER_ID}:${BRANCH}" \
@@ -92,14 +102,26 @@ cd "${ROOTDIR}"
# release managers can use the container to do releases
dockerargs=(--rm=true)
-dockerargs+=(-v "${PWD}:/home/${USER_NAME}/yetus${V_OPTS:-}")
dockerargs+=(-w "/home/${USER_NAME}/yetus")
+dockerargs+=(-v "${PWD}:/home/${USER_NAME}/yetus${V_OPTS:-}")
-if [[ -z ${GITHUB_ACTIONS} ]]; then
- dockerargs+=(-v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}")
- dockerargs+=(-v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg")
- dockerargs+=(-v "${HOME}/.ssh:/home/${USER_NAME}/.ssh")
+# maven cache
+if [[ ! -d ${HOME}/.m2 ]]; then
+ mkdir "${HOME}/.m2"
fi
+dockerargs+=(-v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}")
+
+# GPG Signing for dist creation
+if [[ ! -d ${HOME}/.gnupg ]]; then
+ mkdir "${HOME}/.gnupg"
+fi
+dockerargs+=(-v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg${V_OPTS:-}")
+
+# git opertions
+if [[ ! -d ${HOME}/.ssh ]]; then
+ mkdir "${HOME}/.ssh"
+fi
+dockerargs+=(-v "${HOME}/.ssh:/home/${USER_NAME}/.ssh${V_OPTS:-}")
dockerargs+=(-u "${USER_NAME}")
@@ -111,6 +133,8 @@ if [[ "${GITHUB_ACTIONS}" == true ]]; then
echo "::endgroup::"
fi
+set -x
+
docker run -i \
"${dockerargs[@]}" \
"${YETUS_DOCKER_REPO}-build-${USER_ID}:${BRANCH}" "$@"