You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@yetus.apache.org by aw...@apache.org on 2020/10/12 12:16:15 UTC
[yetus] branch main updated: YETUS-1021. Lots of minor
documentation updates post-overhaul (#154)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 1502218 YETUS-1021. Lots of minor documentation updates post-overhaul (#154)
1502218 is described below
commit 15022185f6d8b4ece99409c16d9cc3ed98705d96
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Mon Oct 12 05:16:07 2020 -0700
YETUS-1021. Lots of minor documentation updates post-overhaul (#154)
* Fix various missing link issues
* Fix the missing buf documentation
* Fix missing precommit API docs
* Update to match current state of YETUS-992
* Add some missing documentation so that actions docs
have something to point to
* Add a linkchecker github action + supporting bits to
help prevent link failure in future
* Removed the support for older versions of the docs since
they are no longer supported
---
.github/workflows/linkcheck.yml | 28 ++++++
asf-site-src/Dockerfile | 25 ++++-
asf-site-src/config.rb | 101 ++++++---------------
asf-site-src/source/contribute/website.html.md | 4 +-
.../documentation/in-progress/precommit/apidocs | 2 +-
.../{apidocs.html.md => apidocs-index.html.md} | 0
.../in-progress/precommit/plugins/ant.html.md | 2 +-
.../in-progress/precommit/plugins/autoconf.html.md | 2 +-
.../precommit/plugins/{buf.md => buf.html.md} | 0
.../precommit/plugins/checkstyle.html.md | 2 +-
.../in-progress/precommit/plugins/cmake.html.md | 2 +-
.../in-progress/precommit/plugins/findbugs.html.md | 10 +-
.../in-progress/precommit/plugins/github.html.md | 4 +-
.../in-progress/precommit/plugins/gitlab.html.md | 4 +-
.../in-progress/precommit/plugins/golang.html.md | 4 +-
.../precommit/plugins/golangcilint.html.md | 2 +-
.../in-progress/precommit/plugins/gradle.html.md | 2 +-
.../in-progress/precommit/plugins/jira.html.md | 2 +-
.../in-progress/precommit/plugins/make.html.md | 2 +-
.../in-progress/precommit/plugins/maven.html.md | 2 +-
.../precommit/robots/githubactions.html.md | 50 +++++-----
.../in-progress/precommit/robots/index.html.md | 39 ++++++--
.../in-progress/precommit/robots/jenkins.html.md | 2 +-
.../in-progress/precommit/robots/travisci.html.md | 2 +-
.../in-progress/precommit/usage-intro.html.md | 19 +++-
asf-site-src/source/yetus-docker-image.html.md | 2 +-
start-build-env.sh | 41 +++++++--
website-tester.sh | 59 ++++++++++++
28 files changed, 272 insertions(+), 142 deletions(-)
diff --git a/.github/workflows/linkcheck.yml b/.github/workflows/linkcheck.yml
new file mode 100644
index 0000000..035aa63
--- /dev/null
+++ b/.github/workflows/linkcheck.yml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+name: linkcheck
+
+on: [push, pull_request] # yamllint disable-line rule:truthy
+
+jobs:
+ linkcheck:
+ runs-on: ubuntu-latest
+ steps:
+ - name: checkout repo
+ uses: actions/checkout@v2
+ - name: test
+ run: ./start-build-env.sh ./website-tester.sh
diff --git a/asf-site-src/Dockerfile b/asf-site-src/Dockerfile
index b0f3491..d9c5788 100644
--- a/asf-site-src/Dockerfile
+++ b/asf-site-src/Dockerfile
@@ -28,6 +28,7 @@ ARG USER_NAME
# hadolint ignore=DL3008
RUN apt-get -q update \
&& apt-get -q install --no-install-recommends -y \
+ apache2 \
pinentry-curses \
pinentry-tty \
subversion \
@@ -46,6 +47,23 @@ ENV HOME /home/${USER_NAME}
WORKDIR /home/${USER_NAME}
+ENV APACHE_PID_FILE /tmp/website/pid
+ENV APACHE_LOG_DIR /tmp/website/logdir
+ENV APACHE_RUN_DIR /tmp/website/rundir
+ENV APACHE_RUN_USER ${USER_NAME}
+ENV APACHE_RUN_GROUP ${USER_NAME}
+
+RUN mkdir -p /tmp/website/{logdir,rundir} \
+ && chown -R ${USER_ID}:${GROUP_ID} /var/www/html /tmp/website \
+ && ln -s /var/www/html /tmp/website/html \
+ && echo "ServerName localhost" >> /etc/apache2/apache2.conf \
+ && sed -i "s,Listen 80,Listen 8123," /etc/apache2/ports.conf \
+ && sed -i "s,:80,:8123," /etc/apache2/sites-available/000-default.conf
+
+# there is no version to pin here, so....
+# hadolint ignore=DL3013
+RUN pip3 install git+https://github.com/linkchecker/linkchecker.git
+
USER ${USER_NAME}
RUN bundle config --global github.https true
@@ -59,9 +77,10 @@ RUN gem install bundler \
middleman:'4.3.11' \
middleman-livereload \
middleman-syntax \
- therubyracer \
- tzinfo-data \
+ nokogiri:1.10.10 \
rake:13.0.1 \
- nokogiri:1.10.10
+ sassc:2.4.0 \
+ therubyracer \
+ tzinfo-data
CMD ["/bin/bash"]
diff --git a/asf-site-src/config.rb b/asf-site-src/config.rb
index e95d86f..b163495 100644
--- a/asf-site-src/config.rb
+++ b/asf-site-src/config.rb
@@ -111,10 +111,10 @@ def releasenotes(output, version)
# file timestamp
puts("Calling releasenotes #{version} @ #{output}")
`(cd #{output} && #{RELEASEDOCMAKER} --project=YETUS --version=#{version} \
- --projecttitle="Apache Yetus" \
- --dirversions --empty \
- --extension=.html.md \
- --usetoday --license --lint=all)`
+ --projecttitle="Apache Yetus" \
+ --dirversions --empty \
+ --extension=.html.md \
+ --usetoday --license --lint=all)`
errmsg = $stderr
return if $CHILD_STATUS.exitstatus.zero?
@@ -122,56 +122,25 @@ def releasenotes(output, version)
abort("releasedocmaker failed to generate release notes for #{version}.")
end
-GITREPO = 'https://github.com/apache/yetus.git'
-
-def build_release_docs(output, version) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+def build_release_docs(output, version)
# TODO: get the version date from jira and do an up to date check instead of building each time.
puts "Building docs for release #{version}"
puts "\tcleaning up output directories in #{output}"
FileUtils.rm_rf("#{output}/build-#{version}", secure: true)
FileUtils.rm_rf("#{output}/#{version}", secure: true)
- if version =~ /^0.[0-8]\./
- puts "\tcloning from tag."
- `(cd "#{output}" && \
- git clone --depth 1 --branch "rel/#{version}" --single-branch -- \
- "#{GITREPO}" "build-#{version}" \
- ) >"#{output}/#{version}_git_checkout.log" 2>&1`
- abort("building docs failed to for #{version}.") unless $CHILD_STATUS.exitstatus.zero?
- puts "\tsetting up markdown docs"
- FileUtils.mkdir "#{output}/#{version}"
- FileUtils.mv(
- Dir.glob("#{output}/build-#{version}/asf-site-src/source/documentation/in-progress/*.md*"),
- "#{output}/#{version}/"
- )
- FileUtils.mv(
- "#{output}/build-#{version}/asf-site-src/source/documentation/in-progress.html.md",
- "#{output}/#{version}.html.md"
- )
- FileUtils.mkdir "#{output}/#{version}/precommit-apidocs"
- precommit_shelldocs(
- "#{output}/#{version}/precommit-apidocs",
- "#{output}/build-#{version}/precommit"
- )
- puts "\tgenerating javadocs"
- `(cd "#{output}/build-#{version}/audience-annotations-component" && mvn -DskipTests -Pinclude-jdiff-module javadoc:aggregate) >"#{output}/#{version}_mvn.log" 2>&1` # rubocop:disable Layout/LineLength
- unless $CHILD_STATUS.exitstatus.zero?
- puts "\tgenerating javadocs failed. maybe maven isn't installed? look in #{output}/#{version}_mvn.log" # rubocop:disable Layout/LineLength
- end
- else
- puts "Downloading and extracting #{version} from ASF archives"
- `(pushd #{output} \
- && mkdir -p build-#{version} \
- && curl --fail --location --output site-#{version}.tar.gz \
- https://archive.apache.org/dist/yetus/#{version}/apache-yetus-#{version}-site.tar.gz \
- && tar -C build-#{version} \
- --strip-components 3 -xzpf site-#{version}.tar.gz \
- apache-yetus-#{version}-site/documentation/in-progress/ \
- && popd
+
+ puts "Downloading and extracting #{version} from ASF archives"
+ `(cd #{output} \
+ && mkdir -p build-#{version} \
+ && curl --fail --location --output site-#{version}.tar.gz \
+ https://archive.apache.org/dist/yetus/#{version}/apache-yetus-#{version}-site.tar.gz \
+ && tar -C build-#{version} \
+ --strip-components 3 -xzpf site-#{version}.tar.gz \
+ apache-yetus-#{version}-site/documentation/in-progress/ \
)`
- puts "Removing #{output}/build-#{version}/CHANGELOG"
- FileUtils.rm_rf("#{output}/build-#{version}/CHANGELOG", secure: true)
- FileUtils.rm_rf("#{output}/build-#{version}/RELEASENOTES", secure: true)
- end
+ puts "Removing #{output}/build-#{version}/CHANGELOG"
+ FileUtils.rm_rf("#{output}/build-#{version}/CHANGELOG", secure: true)
+ FileUtils.rm_rf("#{output}/build-#{version}/RELEASENOTES", secure: true)
end
def precommit_shelldocs(apidocs_dir, source_dir)
@@ -199,7 +168,7 @@ after_configuration do # rubocop:disable Metrics/BlockLength
end
end
- # For Audiene Annotations we just rely on having made javadocs with Maven
+ # For Audience Annotations we just rely on having made javadocs with Maven
sitemap.register_resource_list_manipulator(
:audience_annotations,
ApiDocs.new(
@@ -221,32 +190,22 @@ after_configuration do # rubocop:disable Metrics/BlockLength
app.data.versions.releases&.each do |release|
build_release_docs('target', release)
releasenotes('target', release)
- if release =~ /^0.[0-8]\./
- # stitch the javadoc in place
- sitemap.register_resource_list_manipulator(
- "#{release}_javadocs".to_sym,
- ApiDocs.new(
- sitemap,
- "documentation/#{release}/audience-annotations-apidocs",
- File.expand_path("target/build-#{release}/audience-annotations-component/target/site/apidocs", # rubocop:disable Layout/LineLength
- File.dirname(__FILE__))
- )
- )
-
- else
- sitemap.register_resource_list_manipulator(
- "#{release}_javadocs".to_sym,
- ApiDocs.new(
- sitemap,
- "documentation/#{release}",
- File.expand_path("target/build-#{release}",
- File.dirname(__FILE__))
- )
+ sitemap.register_resource_list_manipulator(
+ "#{release}_javadocs".to_sym,
+ ApiDocs.new(
+ sitemap,
+ "documentation/#{release}",
+ File.expand_path("target/build-#{release}",
+ File.dirname(__FILE__))
)
- end
+ )
end
end
after_build do
File.rename 'target/site/.htaccess.apache', 'target/site/.htaccess'
+ File.rename(
+ 'target/site/documentation/in-progress/precommit/apidocs-index/index.html',
+ 'target/site/documentation/in-progress/precommit/apidocs/index.html'
+ )
end
diff --git a/asf-site-src/source/contribute/website.html.md b/asf-site-src/source/contribute/website.html.md
index bffe4cd..bca25ed 100644
--- a/asf-site-src/source/contribute/website.html.md
+++ b/asf-site-src/source/contribute/website.html.md
@@ -45,7 +45,7 @@ $ cd asf-site-src
$ bundle install
```
-and a working Python 2.7 environment for [releasedocmaker](../in-progress/releasedocmaker/).
+and a working Python 2.7 environment for [releasedocmaker](/documentation/in-progress/releasedocmaker/).
## Make changes in asf-site-src/source
@@ -119,5 +119,5 @@ $ # Finally publish
$ git push origin asf-site
```
-Publishing the website should be possible from the HEAD of the main branch under most circumstances. (See the [Guide for Release Managing](releases) for a notable time period where this won't be true.)
+Publishing the website should be possible from the HEAD of the main branch under most circumstances. (See the [Guide for Release Managing](../releases) for a notable time period where this won't be true.)
Documentation changes will be reviewed as they make their way into the main branch; updates to the `asf-site` branch are handled without further review.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/apidocs b/asf-site-src/source/documentation/in-progress/precommit/apidocs
index 85939e6..38e80e1 120000
--- a/asf-site-src/source/documentation/in-progress/precommit/apidocs
+++ b/asf-site-src/source/documentation/in-progress/precommit/apidocs
@@ -1 +1 @@
-../../../target/in-progress/precommit-apidocs/
\ No newline at end of file
+../../../../target/in-progress/precommit/apidocs/
\ No newline at end of file
diff --git a/asf-site-src/source/documentation/in-progress/precommit/apidocs.html.md b/asf-site-src/source/documentation/in-progress/precommit/apidocs-index.html.md
similarity index 100%
rename from asf-site-src/source/documentation/in-progress/precommit/apidocs.html.md
rename to asf-site-src/source/documentation/in-progress/precommit/apidocs-index.html.md
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/ant.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/ant.html.md
index cb52335..a3bc15b 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/ant.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/ant.html.md
@@ -50,4 +50,4 @@ Provides support for the [Apache Ant](https://ant.apache.org) build tool.
# Developer Notes
-See [build tools](../buildtools) for more information.
+See [build tools](../../buildtools) for more information.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/autoconf.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/autoconf.html.md
index c0076be..e847562 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/autoconf.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/autoconf.html.md
@@ -45,4 +45,4 @@ None
# Developer Notes
-See [build tools](../buildtools) for more information.
+See [build tools](../../buildtools) for more information.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/buf.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/buf.html.md
similarity index 100%
rename from asf-site-src/source/documentation/in-progress/precommit/plugins/buf.md
rename to asf-site-src/source/documentation/in-progress/precommit/plugins/buf.html.md
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/checkstyle.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/checkstyle.html.md
index 7c37925..b9070f7 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/checkstyle.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/checkstyle.html.md
@@ -28,7 +28,7 @@ Test
# Description
[checkstyle](https://checkstyle.sourceforge.net/) is a style linter for Java and many Java-adjacent languages. This plugin requires
-support from the [Build Tool](../buildtools) in order to work correctly. At present, support for `ant`, `maven`, and `gradle` are
+support from the [Build Tool](../../buildtools) in order to work correctly. At present, support for `ant`, `maven`, and `gradle` are
built-in.
# Environment Variables
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/cmake.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/cmake.html.md
index 97974e7..edaa1d6 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/cmake.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/cmake.html.md
@@ -54,4 +54,4 @@ There are two variables that personalities should probably set that correspond t
| CMAKE_BUILD_DIR | `--cmake-build-dir=<path>` |
| CMAKE_ROOT_BUILD | `--cmake-root-build=<bool>` |
-See [build tools](../buildtools) for more information.
+See [build tools](../../buildtools) for more information.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/findbugs.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/findbugs.html.md
index 77cf716..9313d1d 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/findbugs.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/findbugs.html.md
@@ -29,20 +29,20 @@ Test
Provides support for the [FindBugs](https://findbugs.sourceforge.net/) Java utility.
-NOTE: This plug-in is deprecated in favor of using [SpotBugs](spotbugs).
+NOTE: This plug-in is deprecated in favor of using [SpotBugs](../spotbugs).
# Environment Variables
-See the [spotbugs](spotbugs) plugin for more information.
+See the [spotbugs](../spotbugs) plugin for more information.
# Options
-See the [spotbugs](spotbugs) plugin for more information.
+See the [spotbugs](../spotbugs) plugin for more information.
# Docker Notes
-See the [spotbugs](spotbugs) plugin for more information.
+See the [spotbugs](../spotbugs) plugin for more information.
# Developer Notes
-See the [spotbugs](spotbugs) plugin for more information.
+See the [spotbugs](../spotbugs) plugin for more information.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md
index fb0d916..8b21172 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/github.html.md
@@ -27,7 +27,7 @@ Bug System
# Description
-Enables support for reading and writing back to [GitHub](https://github.com/) and compatible systems such as GitHub Enterprise. See also the [Bugsystems](../bugsystems) documentation for more information.
+Enables support for reading and writing back to [GitHub](https://github.com/) and compatible systems such as GitHub Enterprise. See also the [Bugsystems](../../bugsystems) documentation for more information.
# Environment Variables
@@ -49,4 +49,4 @@ None
# Developer Notes
-All of the command line settings may also be set via internal environment variables. However care must be taken to not step on [robots](../robots) that will also set some of these variables.
+All of the command line settings may also be set via internal environment variables. However care must be taken to not step on [robots](../../robots) that will also set some of these variables.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/gitlab.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/gitlab.html.md
index 9c718a2..21bbab1 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/gitlab.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/gitlab.html.md
@@ -27,7 +27,7 @@ Bug System
# Description
-Enables support for reading and writing back to [Gitlab](https://gitlab.com/) and compatible systems such as Gitlab Enterprise. See also the [Bugsystems](../bugsystems) documentation for more information.
+Enables support for reading and writing back to [Gitlab](https://gitlab.com/) and compatible systems such as Gitlab Enterprise. See also the [Bugsystems](../../bugsystems) documentation for more information.
# Environment Variables
@@ -48,4 +48,4 @@ None
# Developer Notes
-All of the command line settings may also be set via internal environment variables. However care must be taken to not step on [robots](../robots) that will also set some of these variables.
+All of the command line settings may also be set via internal environment variables. However care must be taken to not step on [robots](../../robots) that will also set some of these variables.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/golang.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/golang.html.md
index 66b1b28..2b86fca 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/golang.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/golang.html.md
@@ -29,9 +29,9 @@ Test (Compile)
NOTE: Go support is experimental.
-Provides support for [Go](https://golang.com) v1.12 and higher. This support includes the ability to read compiler errors as well as supplemental routines for other precommit tests that support Go directly such as [revive](revive).
+Provides support for [Go](https://golang.com) v1.12 and higher. This support includes the ability to read compiler errors as well as supplemental routines for other precommit tests that support Go directly such as [revive](../revive).
-Amongst other missing features, it does not support using Go as a [build tool](../buildtools) or `go test` formatted output. Additionally, if Go source is detected, `precommit` will use `git checkout` in addition to `git clean` to maintain the source tree as part of the compile cycle.
+Amongst other missing features, it does not support using Go as a [build tool](../../buildtools) or `go test` formatted output. Additionally, if Go source is detected, `precommit` will use `git checkout` in addition to `git clean` to maintain the source tree as part of the compile cycle.
# Environment Variables
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/golangcilint.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/golangcilint.html.md
index 89cd8b1..fdd6acf 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/golangcilint.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/golangcilint.html.md
@@ -31,7 +31,7 @@ NOTE: Go support is experimental. Additionally, this plug-in only supports Go u
Provides support for
[golangci-lint](https://github.com/golangci/golangci-lint). This
-plug-in requires the [golang](golang) plug-in to also be enabled.
+plug-in requires the [golang](../golang) plug-in to also be enabled.
# Environment Variables
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/gradle.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/gradle.html.md
index 0ef2fa6..9fc5567 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/gradle.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/gradle.html.md
@@ -46,4 +46,4 @@ None
# Developer Notes
-See [build tools](../buildtools) for more information.
+See [build tools](../../buildtools) for more information.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/jira.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/jira.html.md
index 191c52e..9e2059e 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/jira.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/jira.html.md
@@ -27,7 +27,7 @@ Bug System
# Description
-Enables support for reading and writing back to [JIRA](https://www.atlassian.com/software/jira), both on-premise and in-cloud. See also the [Bugsystems](../bugsystems) documentation for more information.
+Enables support for reading and writing back to [JIRA](https://www.atlassian.com/software/jira), both on-premise and in-cloud. See also the [Bugsystems](../../bugsystems) documentation for more information.
# Environment Variables
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/make.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/make.html.md
index c633893..1a7b7f8 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/make.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/make.html.md
@@ -54,4 +54,4 @@ None
# Developer Notes
-See [build tools](../buildtools) for more information.
+See [build tools](../../buildtools) for more information.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/maven.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/maven.html.md
index 1806f37..9271830 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/maven.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/maven.html.md
@@ -51,4 +51,4 @@ None
The `maven` plug-in adds several API calls that other plug-ins may use to configure specific Apache Maven behavior, generally around Maven cache management.
-See [build tools](../buildtools) for more information.
+See [build tools](../../buildtools) for more information.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md b/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md
index c120d3c..40c57d3 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/robots/githubactions.html.md
@@ -27,9 +27,9 @@ if `--linecomments` has `github` as a configured bug system (the default).
## Workflow Action
-The Apache Yetus community makes available a built-in action that may be executed as part of a standard GitHub Action workflow.
-The basic workflow file should look like this, changing [VERSION] to either be a released version (or `main` to use the
-bleeding edge, untested, and potentially unstable release):
+The Apache Yetus community makes available a built-in action that may be executed as part of a
+standard GitHub Action workflow. The basic workflow file should look like this, changing [VERSION] to
+either be a released version (or `main` to use the bleeding edge, untested, and potentially unstable release):
```yaml
---
@@ -65,30 +65,36 @@ jobs:
Currently, not all arguments and parameters that can be set on the `test-patch` command line are available to set via the workflow action.
Options currently supported are:
-| Option | Notes | Default |
-|:-------:|:----------------------------:|:-------:|
-| basedir | same as `--basedir` | NONE |
-| buildtool | same as `--build-tool` | `nobuild` |
-| continuousimprovement | same as `--continuous-improvement` | true |
-| excludes | same as `--excludes` | `.yetus-excludes.txt` |
-| githubtoken | same as `--github-token` | NONE |
-| patchdir | same as `--patch-dir` | NONE |
-| pip | same as `--pylint-pip` | pip3 |
-| plugins | same as `--plugins` | all,-asflicense,-author,-findbugs,-gitlabcilint,-shelldocs |
-| pylint | same as `--pylint` | pylint3 |
-| reapermode | same as `--reapermode` | kill |
+| Option | Notes | Default | More Information |
+|:-------:|:----------------------------:|:-------:|:----------------:|
+| basedir | same as `--basedir` | NONE | [Usage Introduction](../../usage-intro) |
+| blankseolignorefile | same as `--blanks-eol-ignore-file` | `.yetus/blanks-eol.txt` | [blanks plug-in](../../plugins/blanks) |
+| blankstabsignorefile | same as `--blanks-tabs-ignore-file` | `.yetus/blanks-tabs.txt` | [blanks plug-in](../../plugins/blanks) |
+| bufbasedir | same as `--buf-basedir` | `.` | [buf plug-in](../../plugins/buf) |
+| buildtool | same as `--build-tool` | `nobuild` | [Build Tools](../../buildtools) |
+| continuousimprovement | same as `--continuous-improvement` | true | [Robots](..) |
+| excludes | same as `--excludes` | `.yetus/excludes.txt` | [Usage Introduction](../../usage-intro) |
+| githubtoken | same as `--github-token` | NONE | [GitHub plug-in](../../plugins/github) |
+| javahome | same as `--java-home` | `/usr/lib/jvm/java-11-openjdk-amd64` | [Java-related plug-ins](../../plugins/javac) |
+| patchdir | same as `--patch-dir` | NONE |[Usage Introduction](../../usage-intro) |
+| pip | same as `--pylint-pip` | pip3 | [pylint plug-in](../../plugins/pylint) |
+| plugins | same as `--plugins` | all,-asflicense,-author,-findbugs,-gitlabcilint,-shelldocs | [Usage Introduction](../../usage-intro) |
+| project | same as `--project` | Auto-set based upon the repository name | [Usage Introduction](../../usage-intro) |
+| pylint | same as `--pylint` | pylint3 | [pylint plug-in](../../plugins/pylint) |
+| reapermode | same as `--reapermode` | kill | [Advanced Usage](../../advanced) |
+| reviveconfig | same as `--revive-config` | `.revive.toml` | [revive plug-in](../../plugins/revive) |
Items marked NONE *MUST* be provided in the workflow yaml file.
Some options are hard-coded to make `test-patch` easier to use:
-| Argument | Value |
-|:--------:|:------:|
-| `--brief-report-file` | patchdir/brief.txt |
-| `--console-report-file` | patchdir/console.txt |
-| `--html-report-file` | patchdir/report.html |
-| `--junit-report-xml` | patchdir/junit-report.xml |
-| `--pylint-requirements` | true |
+| Argument | Value | More Information |
+|:--------:|:------:|:----------------:|
+| `--brief-report-file` | patchdir/brief.txt | [briefreport plug-in](../../plugins/briefreport) |
+| `--console-report-file` | patchdir/console.txt | [QBT](../../qbt) |
+| `--html-report-file` | patchdir/report.html | [htmlout plug-in](../../plugins/htmlout) |
+| `--junit-report-xml` | patchdir/junit-report.xml | [junit plug-in](../../plugins/junit-bugsystem) |
+| `--pylint-requirements` | true | [pylint plug-in](../../plugins/pylint) |
## Manual Configuration
diff --git a/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md b/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md
index 77a384c..6f63bca 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/robots/index.html.md
@@ -25,6 +25,7 @@
* [Automated Robot Detection](#automated-robot-detection)
* [Manual Configuration](#manual-configuration)
* [Sentinel Mode](#sentinel-mode)
+* [Continuous Improvement](#continuous-improvement)
<!-- /MarkdownTOC -->
@@ -45,14 +46,14 @@
The following systems are detected automatically when run in one of these environments:
-* [Azure Pipelines](robots/azurepipelines)
-* [Circle CI](robots/circleci)
-* [Cirrus CI](robots/cirrusci)
-* [GitHub Actions](robots/githubactions)
-* [Gitlab CI](robots/gitlabci)
-* [Jenkins](robots/jenkins)
-* [Semaphore CI](robots/semaphoreci)
-* [Travis CI](robots/travisci)
+* [Azure Pipelines](azurepipelines)
+* [Circle CI](circleci)
+* [Cirrus CI](cirrusci)
+* [GitHub Actions](githubactions)
+* [Gitlab CI](gitlabci)
+* [Jenkins](jenkins)
+* [Semaphore CI](semaphoreci)
+* [Travis CI](travisci)
# Manual Configuration
@@ -76,3 +77,25 @@ If `--robot` is specified without an instance, a random number is generated and
# Sentinel Mode
If stuck Docker containers are a problem, a more aggressive robot may be enabled with the `--sentinel` option. This option enables killing containers that have been running for over 24 hours as well.
+
+# Continuous Improvement
+
+It isn't unusual when adding a new linter to an existing source repository to be overwhelmed with new errors.
+Most projects want to start out where all new patches cannot add new errors, but the existing ones are
+acceptable. In order to enable branches to build with errors but patches/PRs to fail on robots,
+`test-patch` supports the `--continuous-improvement` flag:
+
+Normally, `test-patch` execute with this flag set to false:
+
+```bash
+$ test-patch --continuous-improvement=false (other options)
+```
+
+This setting will fail branches and PRs if any errors are found. However, if this value is set to true:
+
+```bash
+$ test-patch --continuous-improvement=true (other options)
+```
+
+then branches will still report errors but the return code will be successful. Using this mode is ideal for
+many CI/CD systems that are using repositories with known problems.
diff --git a/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md b/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md
index 454c28a..8303808 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/robots/jenkins.html.md
@@ -74,7 +74,7 @@ Experience has shown that certain Jenkins + Java + OS combinations have problems
See also:
* Apache Yetus' source tree [Jenkinsfile](https://github.com/apache/yetus/blob/main/Jenkinsfile) for some tips and tricks.
-* [precommit-admin](../admin), for special utilities built for Jenkins.
+* [precommit-admin](../../admin), for special utilities built for Jenkins.
* [GitHub Branch Source Plugin](https://wiki.jenkins.io/display/JENKINS/GitHub+Branch+Source+Plugin)
* [GitHub Pull Request Builder Plugin](https://wiki.jenkins.io/display/JENKINS/GitHub+pull+request+builder+plugin)
* `https://{your local server}/env-vars.html/`
diff --git a/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md b/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
index 6ade830..82370f4 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/robots/travisci.html.md
@@ -27,7 +27,7 @@ If `${ARTIFACTS_PATH}` is configured, then `--patch-dir` is set to the first lis
Personalities will override the auto-detected Github repository information. It may be necessary to manually configure it in your `.travis.yml` file.
-As of this writing, it is not possible to make the Travis CI build environment use the Apache Yetus pre-built docker images without using `docker run` in the before_install phase. Therefore, using the image is the same as described in the [Apache Yetus Docker Hub Images](/yetus-docker-image) page.
+As of this writing, it is not possible to make the Travis CI build environment use the Apache Yetus pre-built docker images without using `docker run` in the before_install phase. Therefore, using the image is the same as described in the [Apache Yetus Docker Hub Images](../../../../../yetus-docker-image) page.
See also:
diff --git a/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md b/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md
index 8f73ffb..6fb48c0 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/usage-intro.html.md
@@ -35,6 +35,7 @@
* [GITHUB](#github)
* [GITLAB](#gitlab)
* [Generic URLs](#generic-urls)
+* [Excluding Files](#excluding-files)
* [Project-specific Capabilities](#project-specific-capabilities)
* [Direct Method](#direct-method)
* [Project Method](#project-method)
@@ -369,7 +370,21 @@ For example:
$ test-patch --plugins=all https://example.com/webserver/file.patch
```
-... will download and process the file.patch from the example.com webserver.
+process the file.patch from the example.com webserver.
+
+# Excluding Files
+
+Some repositories have content that is either imported from other sources (aka "vendored") or in some
+other way have files that are known to break tests. By default, `.yetus/excludes.txt` will be read for any
+file, directory, or extended regular expression as the list to remove from test results. The file to read
+may be overwritten by using the `--excludes` flag:
+
+```bash
+$ test-patch --plugins=all --excludes=(filename) (other options)
+```
+
+ NOTE: for best results, avoid using ^ and $ in your regular expressions. `test-patch` will automatically
+ anchor the contents of that file.
# Project-specific Capabilities
@@ -423,7 +438,7 @@ NOTE: JAVA\_HOME is always appended to the list of JDKs in MultiJDK mode. If JA
$ test-patch --docker
```
-This command will do some preliminary setup and then re-execute itself inside a Docker container. For more information on how to provide a custom Dockerfile and other Docker-specific features, see the specific [precommit Docker support](../docker) page and the [Apache Yetus Docker Hub Images](/yetus-docker-image) page for more information on the convenience Docker images.
+This command will do some preliminary setup and then re-execute itself inside a Docker container. For more information on how to provide a custom Dockerfile and other Docker-specific features, see the specific [precommit Docker support](../docker) page and the [Apache Yetus Docker Hub Images](../../../../yetus-docker-image) page for more information on the convenience Docker images.
# In Closing
diff --git a/asf-site-src/source/yetus-docker-image.html.md b/asf-site-src/source/yetus-docker-image.html.md
index f7cea5d..1d99c4b 100644
--- a/asf-site-src/source/yetus-docker-image.html.md
+++ b/asf-site-src/source/yetus-docker-image.html.md
@@ -89,4 +89,4 @@ This example builds a docker image based off of Apache Yetus 0.9.0 but with the
# A Note About Precommit
-test-patch and friends have direct support for Docker outside of the convenience images. That information is covered in-depth in the[precommit-docker](../precommit-docker) section.
+test-patch and friends have direct support for Docker outside of the convenience images. That information is covered in-depth in the[precommit-docker](/documentation/in-progress/precommit/docker/) section.
diff --git a/start-build-env.sh b/start-build-env.sh
index c73d22f..1b8af64 100755
--- a/start-build-env.sh
+++ b/start-build-env.sh
@@ -15,7 +15,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e # exit on error
+if [[ "${GITHUB_ACTIONS}" == true ]]; then
+ echo "::group::start-build-env"
+fi
+
+set -e # exit on error
ROOTDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")" >/dev/null && pwd -P)
YETUS_DOCKER_REPO=${YETUS_DOCKER_REPO:-apache/yetus}
@@ -23,10 +27,11 @@ YETUS_DOCKER_REPO=${YETUS_DOCKER_REPO:-apache/yetus}
# moving to the path of the Dockerfile reduces the context
cd "${ROOTDIR}/precommit/src/main/shell/test-patch-docker"
-BRANCH=$(git branch | grep '\*' | cut -d ' ' -f2)
+BRANCH=$(git branch | grep '\*' | cut -d ' ' -f2 )
if [[ "${BRANCH}" =~ HEAD ]]; then
BRANCH=$(git branch | grep '\*' | awk '{print $NF}' | sed -e s,rel/,,g -e s,\),,g )
fi
+BRANCH=${BRANCH//\//_}
echo "Attempting a few pulls of ${YETUS_DOCKER_REPO} and ${YETUS_DOCKER_REPO}-base to save time"
echo "Errors here will be ignored!"
@@ -34,7 +39,7 @@ docker pull "${YETUS_DOCKER_REPO}-base:${BRANCH}" || docker pull "${YETUS_DOCKER
docker pull "${YETUS_DOCKER_REPO}:${BRANCH}" || docker pull "${YETUS_DOCKER_REPO}:main" || true
docker build \
- --cache-from="${YETUS_DOCKER_REPO}-base:${BRANCH},${YETUS_DOCKER_REPO}-base:main,${YETUS_DOCKER_REPO}:${BRANCH},${YETUS_DOCKER_REPO}:main" \
+ --cache-from="${YETUS_DOCKER_REPO}-build:${BRANCH},${YETUS_DOCKER_REPO}-base:${BRANCH},${YETUS_DOCKER_REPO}-base:main,${YETUS_DOCKER_REPO}:${BRANCH},${YETUS_DOCKER_REPO}:main" \
-t "${YETUS_DOCKER_REPO}-build:${BRANCH}" .
USER_NAME=${SUDO_USER:=$USER}
@@ -85,11 +90,27 @@ cd "${ROOTDIR}"
# builds because the dependencies are downloaded only once.
# Additionally, we mount GPG and SSH directories so that
# release managers can use the container to do releases
-docker run --rm=true -i -t \
- -v "${PWD}:/home/${USER_NAME}/yetus${V_OPTS:-}" \
- -w "/home/${USER_NAME}/yetus" \
- -v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \
- -v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg" \
- -v "${HOME}/.ssh:/home/${USER_NAME}/.ssh" \
- -u "${USER_NAME}" \
+
+dockerargs=(--rm=true)
+dockerargs+=(-v "${PWD}:/home/${USER_NAME}/yetus${V_OPTS:-}")
+dockerargs+=(-w "/home/${USER_NAME}/yetus")
+
+if [[ -z ${GITHUB_ACTIONS} ]]; then
+ dockerargs+=(-v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}")
+ dockerargs+=(-v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg")
+ dockerargs+=(-v "${HOME}/.ssh:/home/${USER_NAME}/.ssh")
+fi
+
+dockerargs+=(-u "${USER_NAME}")
+
+if tty -s; then
+ dockerargs+=(-t)
+fi
+
+if [[ "${GITHUB_ACTIONS}" == true ]]; then
+ echo "::endgroup::"
+fi
+
+docker run -i \
+ "${dockerargs[@]}" \
"${YETUS_DOCKER_REPO}-build-${USER_ID}:${BRANCH}" "$@"
diff --git a/website-tester.sh b/website-tester.sh
new file mode 100755
index 0000000..c1bf2c6
--- /dev/null
+++ b/website-tester.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+declare -i result
+
+echo "::group::mvn install"
+mvn --batch-mode install
+echo "::endgroup::"
+
+echo "::group::mvn site"
+mvn --batch-mode site site:stage </dev/null
+echo "::endgroup::"
+
+echo "::group::tar"
+tar -C /tmp/website/html --strip-components 1 \
+ -xpf yetus-dist/target/artifacts/apache-yetus-*-SNAPSHOT-site.tar.gz
+echo "::endgroup::"
+
+echo "::group::start apache httpd"
+apache2
+echo "::endgroup::"
+
+echo "::group::linkchecker"
+linkchecker \
+ --ignore-url 'http://localhost:8123/documentation/[0-9]+.[0-9]+.[0-9]+' \
+ --ignore-url 'resources/fonts/dejavu.css' \
+ --no-robots --file-output=csv \
+ http://localhost:8123
+result=$?
+echo "::endgroup::"
+
+#
+# urlname;parentname;base;result;warningstring;infostring;valid;url;line;column;name;dltime;size;checktime;cached;level;modified
+# in-page reference: $1
+# generated page: $2
+# error code: $4
+# expected page: $8
+#
+echo "::group::github actions check annotations"
+grep -v '^#' linkchecker-out.csv \
+ | grep -v '^urlname' \
+ | awk '-F;' \
+ '{print "::error::["$4"] mdref: ("$1") | pageref: "$2" | expected page: "$8}'
+echo "::endgroup::"
+exit ${result}