You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2024/06/22 02:38:51 UTC
(hbase) branch branch-2.6 updated: HBASE-28679 Upgrade yetus to a newer version (#6012)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 265dcb47372 HBASE-28679 Upgrade yetus to a newer version (#6012)
265dcb47372 is described below
commit 265dcb47372623e4493f8c4a34cc3d4b16e3176e
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Sat Jun 22 10:22:56 2024 +0800
HBASE-28679 Upgrade yetus to a newer version (#6012)
Signed-off-by: Nick Dimiduk <nd...@apache.org>
(cherry picked from commit 17ce7c3b4c5e58af1f70d17e62940c9dd6fb4660)
---
dev-support/Jenkinsfile | 8 +++---
dev-support/Jenkinsfile_GitHub | 38 ++++++++++++++-------------
dev-support/docker/Dockerfile | 2 ++
dev-support/hbase_nightly_yetus.sh | 10 +++----
dev-support/jenkins_precommit_github_yetus.sh | 13 +++++----
5 files changed, 38 insertions(+), 33 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 19256ccb9b1..e514b31702c 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -31,7 +31,7 @@ pipeline {
disableConcurrentBuilds()
}
environment {
- YETUS_RELEASE = '0.12.0'
+ YETUS_RELEASE = '0.15.0'
// where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
@@ -42,12 +42,12 @@ pipeline {
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
PERSONALITY_FILE = 'tools/personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
- AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
- WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+ AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+ BLANKS_IGNORE_LIST = '.*/generated/.*'
// output from surefire; sadly the archive function in yetus only works on file names.
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
- TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+ TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes"
// TODO does hadoopcheck need to be jdk specific?
SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'
diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
index 44443cd697e..a2b953464c0 100644
--- a/dev-support/Jenkinsfile_GitHub
+++ b/dev-support/Jenkinsfile_GitHub
@@ -36,14 +36,16 @@ pipeline {
YETUS_REL = 'yetus'
DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
- // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
- YETUS_VERSION = 'rel/0.12.0'
+ YETUS_VERSION = '0.15.0'
GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
+ // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
+ AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+ BLANKS_IGNORE_LIST = '.*/generated/.*'
// output from surefire; sadly the archive function in yetus only works on file names.
ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
- TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+ TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
// set build parallel
BUILD_THREAD = 4
@@ -101,11 +103,11 @@ pipeline {
checkout scm
}
dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
- )
+ sh'''#!/usr/bin/env bash
+ wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
}
dir("${WORKDIR}") {
withCredentials([
@@ -228,11 +230,11 @@ pipeline {
checkout scm
}
dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
- )
+ sh'''#!/usr/bin/env bash
+ wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
}
dir("${WORKDIR}") {
withCredentials([
@@ -365,11 +367,11 @@ pipeline {
checkout scm
}
dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
- )
+ sh'''#!/usr/bin/env bash
+ wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ '''
}
dir("${WORKDIR}") {
withCredentials([
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 1f841d1d867..bc8dee088ae 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -53,6 +53,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
ruby=1:3.0* \
ruby-dev=1:3.0* \
shellcheck='0.8.0-*' \
+ libxml2-dev='2.9.13+dfsg-*' \
+ libxml2-utils='2.9.13+dfsg-*' \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* \
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 4c671dcfef6..3423a7e73c5 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -20,7 +20,7 @@ declare -i missing_env=0
# Validate params
for required_env in "TESTS" "PERSONALITY_FILE" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE" \
"OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST" \
- "WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
+ "BLANKS_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
"USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do
if [ -z "${!required_env}" ]; then
echo "[ERROR] Required environment variable '${required_env}' is not set."
@@ -59,8 +59,8 @@ YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-eol-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-tabs-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
@@ -106,11 +106,9 @@ if [[ -n "${JAVA8_HOME}" ]]; then
fi
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
- YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
else
- YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-git/shelldocs/shelldocs.py" "${YETUS_ARGS[@]}")
- TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/test-patch.sh"
+ TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/src/main/shell/test-patch.sh"
fi
echo "Launching yetus with command line:"
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh
index d22351bcf5a..95ddc0ed4aa 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -41,6 +41,8 @@ declare -a required_envs=(
"SOURCEDIR"
"TESTS_FILTER"
"YETUSDIR"
+ "AUTHOR_IGNORE_LIST"
+ "BLANKS_IGNORE_LIST"
)
# Validate params
for required_env in "${required_envs[@]}"; do
@@ -57,7 +59,7 @@ if [ ${missing_env} -gt 0 ]; then
fi
# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
-TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+TESTPATCHBIN="${YETUSDIR}/bin/test-patch"
# this must be clean for every run
rm -rf "${PATCHDIR}"
@@ -87,8 +89,8 @@ YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
# enable writing back to Github
-YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
-YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}")
+YETUS_ARGS+=("--github-write-comment")
# auto-kill any surefire stragglers during unit test runs
YETUS_ARGS+=("--reapermode=kill")
# set relatively high limits for ASF machines
@@ -109,8 +111,9 @@ YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
YETUS_ARGS+=("--mvn-custom-repos")
YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
-YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
-YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
+YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}*")
YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
YETUS_ARGS+=("--quick-hadoopcheck")