You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@yetus.apache.org by aw...@apache.org on 2022/05/02 16:09:00 UTC
[yetus] branch main updated: YETUS-1105. xml check fails since JDK15 (#269)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 50c4bbdb YETUS-1105. xml check fails since JDK15 (#269)
50c4bbdb is described below
commit 50c4bbdb4a1fdf86ab66603c51494298adf97041
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Mon May 2 09:08:56 2022 -0700
YETUS-1105. xml check fails since JDK15 (#269)
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
.../in-progress/precommit/index.html.md | 16 ++-
.../in-progress/precommit/plugins/xml.html.md | 3 +-
precommit/src/main/shell/plugins.d/xml.sh | 152 +++++++++++++++------
.../src/main/shell/test-patch-docker/Dockerfile | 4 +-
4 files changed, 122 insertions(+), 53 deletions(-)
diff --git a/asf-site-src/source/documentation/in-progress/precommit/index.html.md b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
index 77913ab5..34d27dea 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/index.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
@@ -86,7 +86,6 @@ These plug-ins are native to Apace Yetus and are (usually!) always available:
* [pathlen](plugins/pathlen)
* [slack](plugins/slack)
* [unitveto](plugins/unitveto)
-* [xml](plugins/xml)
### Optional Plug-ins
@@ -132,27 +131,30 @@ Compiler Support:
* Java, both [javac](plugins/javac) and [javadoc](plugins/javadoc)
* Scala, both [scalac](plugins/scalac) and [scaladoc](plugins/scaladoc)
+Commonly Parsed File Formats:
+
+* .md: [markdownlint-cli](plugins/markdownlint)
+* json: [jsonlint](plugins/jsonlint)
+* .proto: [buf](plugins/buf)
+* .yaml/.yml: [yamllint](plugins/yamllint)
+* .html/.xml: [xml](plugins/xml)
+
Language Support, Licensing, and more:
-* [ansible-lint](plugins/ansiblelint)
* [Apache Creadur Rat](plugins/asflicense)
-* [buf](plugins/buf)
* [checkmake](plugins/checkmake)
* [checkstyle](plugins/checkstyle)
* [codespell](plugins/codespell)
* [detect-secrets](plugins/detsecrets)
* [golangci-lint](plugins/golangcilint)
-* [hadolint](plugins/hadolint)
+* Dockerfile: [hadolint](plugins/hadolint)
* [jshint](plugins/jshint)
-* [jsonlint](plugins/jsonlint)
-* [markdownlint-cli](plugins/markdownlint)
* [Perl::Critic](plugins/perlcritic)
* [pylint](plugins/pylint)
* [revive](plugins/revive)
* [rubocop](plugins/rubocop)
* [shellcheck](plugins/shellcheck)
* [SpotBugs](plugins/spotbugs)
-* [yamllint](plugins/yamllint)
# Related Utilities
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/xml.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/xml.html.md
index 808fc0c7..aba52cbc 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/xml.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/xml.html.md
@@ -29,8 +29,7 @@ Test
# Description
-Verify if XML files can be parsed. Requires `jrunscript`, which comes as part of many Java
-distributions. See [java](javac.html) for more information on configuring Java.
+Verify if HTML and XML files can be parsed. Requires `xmllint`.
# Environment Variables
diff --git a/precommit/src/main/shell/plugins.d/xml.sh b/precommit/src/main/shell/plugins.d/xml.sh
index 6297dc65..b8f000b3 100755
--- a/precommit/src/main/shell/plugins.d/xml.sh
+++ b/precommit/src/main/shell/plugins.d/xml.sh
@@ -16,76 +16,142 @@
# SHELLDOC-IGNORE
-declare -a XML_FILES
+add_test_type xmllint
-add_test_type xml
+XMLLINT_TIMER=0
-function xml_filefilter
-{
- declare filename=$1
+XMLLINT=${XMLLINT:-$(command -v xmllint 2>/dev/null)}
- if [[ ${filename} =~ \.xml$ ]]; then
- add_test xml
- fi
+function xmllint_usage
+{
+ yetus_add_option "--xmllint=<file>" "path to xmllint executable"
}
-function xml_precheck
+function xmllint_parse_args
{
- if ! verify_command "jrunscript" "${JAVA_HOME}/bin/jrunscript"; then
- add_vote_table_v2 0 xml "" "jrunscript was not available."
- delete_test xml
- fi
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --xmllint=*)
+ delete_parameter "${i}"
+ XMLLINT=${i#*=}
+ ;;
+ esac
+ done
}
-function xml_postcompile
+function xmllint_filefilter
{
- declare repostatus=$1
- declare js
- declare i
- declare count
+ local filename=$1
- if ! verify_needed_test xml; then
- return 0
+ if [[ ${filename} =~ \.xml$ || ${filename} =~ \.html?$ ]]; then
+ add_test xmllint
fi
+}
- if [[ "${repostatus}" = branch ]]; then
+function xmllint_precheck
+{
+ if ! verify_command xmllint "${XMLLINT}"; then
+ add_vote_table_v2 0 xmllint "" "xmllint was not available."
+ delete_test xmllint
return 0
fi
+}
- big_console_header "XML verification: ${BUILDMODE}"
- js="${JAVA_HOME}/bin/jrunscript"
+function xmllint_exec
+{
+ declare i
+ declare repostatus=$1
+ declare tempfile
- start_clock
+ tempfile="${PATCH_DIR}/xmltmp.$$"
pushd "${BASEDIR}" >/dev/null || return 1
+
+ # xmllint's output is _ugly_ and requires filtering.
+
+ printf "^%b:\n" "${CHANGED_FILES[@]}" > "${tempfile}"
+
for i in "${CHANGED_FILES[@]}"; do
if [[ ${i} =~ \.xml$ && -f ${i} ]]; then
- if ! "${js}" -e "XMLDocument(arguments[0])" "${i}" > "${PATCH_DIR}/xml.txt.tmp" 2>&1; then
- {
- echo ""
- echo "${i}:"
- echo ""
- cat "${PATCH_DIR}/xml.txt.tmp"
- } >> "${PATCH_DIR}/xml.txt"
- ((count=count+1))
- XML_FILES+=("${i}")
- fi
+ "${XMLLINT}" --nout "${i}" 2>&1 \
+ | "${GREP}" -E -f "${tempfile}" \
+ >> "${PATCH_DIR}/${repostatus}-xmllint-result.txt"
+ fi
+ if [[ ${i} =~ \.html?$ && -f ${i} ]]; then
+ "${XMLLINT}" --html --nout "${i}" 2>&1 \
+ | "${GREP}" -E -f "${tempfile}" \
+ >> "${PATCH_DIR}/${repostatus}-xmllint-result.txt"
fi
done
-
+ rm "${tempfile}"
popd >/dev/null || return 1
+ return 0
+}
- if [[ -f "${PATCH_DIR}/xml.txt.tmp" ]]; then
- rm "${PATCH_DIR}/xml.txt.tmp"
+function xmllint_preapply
+{
+ if ! verify_needed_test xmllint; then
+ return 0
fi
- if [[ ${count} -gt 0 ]]; then
- add_vote_table_v2 -1 xml "@@BASE@@/xml.txt" "${BUILDMODEMSG} has ${count} ill-formed XML file(s)."
- populate_test_table "XML" "Parsing Error(s):" "${XML_FILES[@]}"
- return 1
- fi
+ big_console_header "xmllint plugin: ${PATCH_BRANCH}"
- add_vote_table_v2 +1 xml "" "${BUILDMODEMSG} has no ill-formed XML file."
+ start_clock
+
+ xmllint_exec branch
+
+ XMLLINT_TIMER=$(stop_clock)
return 0
}
+
+## @description Wrapper to call column_calcdiffs
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param branchlog
+## @param patchlog
+## @return differences
+function xmllint_calcdiffs
+{
+ column_calcdiffs "$@"
+}
+
+function xmllint_postapply
+{
+ if ! verify_needed_test xmllint; then
+ return 0
+ fi
+
+ big_console_header "xmllint plugin: ${BUILDMODE}"
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${XMLLINT_TIMER}"
+
+ xmllint_exec patch
+
+ # shellcheck disable=SC2016
+ XMLLINT_VERSION=$("${XMLLINT}" --version 2>&1 | head -1 | "${AWK}" '{print $NF}')
+ add_version_data xmllint "${XMLLINT_VERSION}"
+
+ root_postlog_compare \
+ xmllint \
+ "${PATCH_DIR}/branch-xmllint-result.txt" \
+ "${PATCH_DIR}/patch-xmllint-result.txt"
+}
+
+function xmllint_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ xmllint_preapply
+ else
+ xmllint_postapply
+ fi
+}
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile
index 3dbf1013..e5e408ca 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -37,7 +37,8 @@ ENV DEBCONF_TERSE true
# Install some basic Apache Yetus requirements
# some git repos need ssh-client so do it too
# Adding libffi-dev for all the programming languages
-# that take advantage of it.
+# that take advantage of it. Also throw in
+# xmllint here since so much links against libxml anyway.
######
# hadolint ignore=DL3008
RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
@@ -50,6 +51,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
gpg \
gpg-agent \
libffi-dev \
+ libxml2-utils \
locales \
pkg-config \
rsync \