You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2021/10/21 14:43:46 UTC
[hbase] branch HBASE-26318 updated (d0e05ef -> a05698a)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch HBASE-26318
in repository https://gitbox.apache.org/repos/asf/hbase.git.
omit d0e05ef use HBASE-26318 script
omit 59f12b3 fix path
omit 7c18a3d disable other checks to speed up test
omit ad6697e add support for nightly
omit 11c16f8 fix base path
omit 36ad7c7 fix
omit 425548a test upload site
add 856d08e HBASE-26371 Prioritize meta region move over other region moves in region_mover (#3767)
add 64eb237 HBASE-26383 HBCK incorrectly reports inconsistencies for recently split regions following a master failover (#3775)
add 169f2cf HBASE-26380 Add 2.3.7 to downloads page (#3770)
add 4838246 HBASE-26385 Clear CellScanner when replay (#3773)
new a05698a HBASE-26362 Upload mvn site artifacts for nightly build to nightlies
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d0e05ef)
\
N -- N -- N refs/heads/HBASE-26318 (a05698a)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/Jenkinsfile | 13 ++-
.../org/apache/hadoop/hbase/master/HbckChore.java | 3 +-
.../hadoop/hbase/regionserver/RSRpcServices.java | 2 +
.../org/apache/hadoop/hbase/util/RegionMover.java | 93 ++++++++++++++++------
src/site/xdoc/downloads.xml | 16 ++--
5 files changed, 86 insertions(+), 41 deletions(-)
[hbase] 01/01: HBASE-26362 Upload mvn site artifacts for nightly
build to nightlies
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch HBASE-26318
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a05698a3122deb2d1bc34c73b450dd3f27253eb2
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed Oct 20 21:57:14 2021 +0800
HBASE-26362 Upload mvn site artifacts for nightly build to nightlies
---
dev-support/Jenkinsfile | 36 ++++++++++++++++++++++++---
dev-support/Jenkinsfile_GitHub | 36 ++++++++++++++++++++++++---
dev-support/hbase-personality.sh | 10 +++++++-
dev-support/hbase_nightly_yetus.sh | 5 ++++
dev-support/jenkins_precommit_github_yetus.sh | 4 +++
5 files changed, 82 insertions(+), 9 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 95659b8..d987182 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -54,6 +54,8 @@ pipeline {
SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'
DEEP_CHECKS = 'compile,htmlout,javac,maven,mvninstall,shadedjars,unit' // run by 'yetus jdkX (HadoopY) checks'
ASF_NIGHTLIES = 'https://nightlies.apache.org'
+ ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
+ ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
}
parameters {
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
@@ -205,6 +207,7 @@ pipeline {
SET_JAVA_HOME = '/usr/lib/jvm/java-8'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
+ ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}"
}
steps {
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
@@ -253,6 +256,31 @@ pipeline {
post {
always {
stash name: 'general-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ sshPublisher(publishers: [
+ sshPublisherDesc(configName: 'Nightlies',
+ transfers: [
+ sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
+ sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/*-site/*,${env.OUTPUT_DIR_RELATIVE}/*-site/**/*"
+ )
+ ]
+ )
+ ])
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/branch-site" ]; then
+ echo "Remove ${OUTPUT_DIR}/branch-site for saving space"
+ rm -rf "${OUTPUT_DIR}/branch-site"
+ python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${OUTPUT_DIR}/branch-site.html"
+ else
+ echo "No branch-site, skipping"
+ fi
+ if [ -d "${OUTPUT_DIR}/patch-site" ]; then
+ echo "Remove ${OUTPUT_DIR}/patch-site for saving space"
+ rm -rf "${OUTPUT_DIR}/patch-site"
+ python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${OUTPUT_DIR}/patch-site.html"
+ else
+ echo "No patch-site, skipping"
+ fi
+ '''
// Has to be relative to WORKSPACE.
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
@@ -358,7 +386,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
- python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -468,7 +496,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
- python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -582,7 +610,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
- python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -698,7 +726,7 @@ pipeline {
if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
rm -rf "${OUTPUT_DIR}/test_logs.zip"
- python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ python ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
index 530fc6d..c9dee54 100644
--- a/dev-support/Jenkinsfile_GitHub
+++ b/dev-support/Jenkinsfile_GitHub
@@ -53,6 +53,8 @@ pipeline {
WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
ASF_NIGHTLIES = 'https://nightlies.apache.org'
+ ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
+ ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
}
parameters {
@@ -83,6 +85,7 @@ pipeline {
BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+ ASF_NIGHTLIES_GENERAL_CHECK_BASE="${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}"
}
steps {
dir("${SOURCEDIR}") {
@@ -124,9 +127,34 @@ pipeline {
}
post {
always {
+ sshPublisher(publishers: [
+ sshPublisherDesc(configName: 'Nightlies',
+ transfers: [
+ sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
+ sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/*,${env.WORKDIR_REL}/${env.PATCH_REL}/*-site/**/*"
+ )
+ ]
+ )
+ ])
+ sh '''#!/bin/bash -e
+ if [ -d "${PATCHDIR}/branch-site" ]; then
+ echo "Remove ${PATCHDIR}/branch-site for saving space"
+ rm -rf "${PATCHDIR}/branch-site"
+ python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/branch-site" > "${PATCHDIR}/branch-site.html"
+ else
+ echo "No branch-site, skipping"
+ fi
+ if [ -d "${PATCHDIR}/patch-site" ]; then
+ echo "Remove ${PATCHDIR}/patch-site for saving space"
+ rm -rf "${PATCHDIR}/patch-site"
+ python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/patch-site" > "${PATCHDIR}/patch-site.html"
+ else
+ echo "No patch-site, skipping"
+ fi
+ '''
// Has to be relative to WORKSPACE.
- archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
- archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*"
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*"
publishHTML target: [
allowMissing: true,
keepAll: true,
@@ -250,7 +278,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip"
- python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+ python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
@@ -381,7 +409,7 @@ pipeline {
if [ -f "${PATCHDIR}/test_logs.zip" ]; then
echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
rm -rf "${PATCHDIR}/test_logs.zip"
- python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+ python ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
else
echo "No test_logs.zip, skipping"
fi
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index 6eb38e3..6de1e06 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -115,6 +115,10 @@ function personality_parse_args
delete_parameter "${i}"
SKIP_ERRORPRONE=true
;;
+ --asf-nightlies-general-check-base=*)
+ delete_parameter "${i}"
+ ASF_NIGHTLIES_GENERAL_CHECK_BASE=${i#*=}
+ ;;
esac
done
}
@@ -414,7 +418,11 @@ function refguide_rebuild
fi
add_vote_table 0 refguide "${repostatus} has no errors when building the reference guide. See footer for rendered docs, which you should manually inspect."
- add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
+ if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
+ add_footer_table refguide "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}/${repostatus}-site/book.html"
+ else
+ add_footer_table refguide "@@BASE@@/${repostatus}-site/book.html"
+ fi
return 0
}
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 65b5270..3fb1ac7 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -96,6 +96,11 @@ if [[ ! -d "${OUTPUT_DIR}" ]]; then
exit 1
fi
+# pass asf nightlies url in
+if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
+ YETUS_ARGS=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" "${YETUS_ARGS[@]}")
+fi
+
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh
index 5bb2b1b..9652d1e 100755
--- a/dev-support/jenkins_precommit_github_yetus.sh
+++ b/dev-support/jenkins_precommit_github_yetus.sh
@@ -143,6 +143,10 @@ fi
YETUS_ARGS+=("--sentinel")
# use emoji vote so it is easier to find the broken line
YETUS_ARGS+=("--github-use-emoji-vote")
+# pass asf nightlies url in
+if [[ -n "${ASF_NIGHTLIES_GENERAL_CHECK_BASE}" ]]; then
+ YETUS_ARGS+=("--asf-nightlies-general-check-base=${ASF_NIGHTLIES_GENERAL_CHECK_BASE}")
+fi
echo "Launching yetus with command line:"
echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"