You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2024/02/29 08:49:27 UTC
(hbase) branch master updated: HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to ni… (#5689)
This is an automated email from the ASF dual-hosted git repository.
rajeshbabu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 8a715fadd58 HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to ni… (#5689)
8a715fadd58 is described below
commit 8a715fadd587dc89d5728090ba17cbad21a74398
Author: Rajeshbabu Chintaguntla <ra...@apache.org>
AuthorDate: Thu Feb 29 14:19:22 2024 +0530
HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to ni… (#5689)
HBASE-27949 [JDK17] Add JDK17 compilation and unit test support to nightly job
HBASE-26372 [JDK17] Jenkins build support
---
dev-support/Jenkinsfile | 119 +++++++++++++++++++++++++++++++++++
dev-support/Jenkinsfile_GitHub | 140 +++++++++++++++++++++++++++++++++++++++++
dev-support/docker/Dockerfile | 15 +++++
3 files changed, 274 insertions(+)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 19256ccb9b1..461df28c8e9 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -37,6 +37,7 @@ pipeline {
OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3'
OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3'
+ OUTPUT_DIR_RELATIVE_JDK17_HADOOP3 = 'output-jdk17-hadoop3'
PROJECT = 'hbase'
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
@@ -188,6 +189,7 @@ pipeline {
stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match"
stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match"
stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match"
+ stash name: 'jdk17-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}/doesn't-match"
stash name: 'srctarball-result', allowEmpty: true, includes: "output-srctarball/doesn't-match"
}
}
@@ -628,6 +630,123 @@ pipeline {
}
}
}
+
+ stage ('yetus jdk17 hadoop3 checks') {
+ when {
+ anyOf {
+ branch 'master';branch 'branch-3'
+ }
+ }
+ agent {
+ node {
+ label 'hbase'
+ }
+ }
+ environment {
+ BASEDIR = "${env.WORKSPACE}/component"
+ TESTS = "${env.DEEP_CHECKS}"
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK17_HADOOP3}"
+ SET_JAVA_HOME = "/usr/lib/jvm/java-17"
+ // Activates hadoop 3.0 profile in maven runs.
+ HADOOP_PROFILE = '3.0'
+ SKIP_ERRORPRONE = true
+ }
+ steps {
+ // Must do prior to anything else, since if one of them timesout we'll stash the commentfile
+ sh '''#!/usr/bin/env bash
+ set -e
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ echo '(x) {color:red}-1 jdk17 hadoop3 checks{color}' >"${OUTPUT_DIR}/commentfile"
+ echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
+ '''
+ unstash 'yetus'
+ dir('component') {
+ checkout scm
+ }
+ sh '''#!/usr/bin/env bash
+ set -e
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
+ '''
+ script {
+ def ret = sh(
+ returnStatus: true,
+ script: '''#!/usr/bin/env bash
+ set -e
+ declare -i status=0
+ if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
+ echo '(/) {color:green}+1 jdk17 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
+ else
+ echo '(x) {color:red}-1 jdk17 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
+ status=1
+ fi
+ echo "-- For more information [see jdk17 report|${BUILD_URL}JDK17_20Nightly_20Build_20Report_20_28Hadoop3_29/]" >> "${OUTPUT_DIR}/commentfile"
+ exit "${status}"
+ '''
+ )
+ if (ret != 0) {
+ // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
+ // test output. See HBASE-26339 for more details.
+ currentBuild.result = 'UNSTABLE'
+ }
+ }
+ }
+ post {
+ always {
+ stash name: 'jdk17-hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
+ // zip surefire reports.
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/archiver" ]; then
+ count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
+ '''
+ sshPublisher(publishers: [
+ sshPublisherDesc(configName: 'Nightlies',
+ transfers: [
+ sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
+ sourceFiles: "${env.OUTPUT_DIR_RELATIVE}/test_logs.zip"
+ )
+ ]
+ )
+ ])
+ // remove the big test logs zip file, store the nightlies url in test_logs.html
+ sh '''#!/bin/bash -e
+ if [ -f "${OUTPUT_DIR}/test_logs.zip" ]; then
+ echo "Remove ${OUTPUT_DIR}/test_logs.zip for saving space"
+ rm -rf "${OUTPUT_DIR}/test_logs.zip"
+ python3 ${BASEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${OUTPUT_DIR_RELATIVE}" > "${OUTPUT_DIR}/test_logs.html"
+ else
+ echo "No test_logs.zip, skipping"
+ fi
+ '''
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
+ archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing : true,
+ keepAll : true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE.
+ reportDir : "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles : 'console-report.html',
+ reportName : 'JDK17 Nightly Build Report (Hadoop3)'
+ ]
+ }
+ }
+ }
+
// This is meant to mimic what a release manager will do to create RCs.
// See http://hbase.apache.org/book.html#maven.release
// TODO (HBASE-23870): replace this with invocation of the release tool
diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
index bf916903411..62b16287e1e 100644
--- a/dev-support/Jenkinsfile_GitHub
+++ b/dev-support/Jenkinsfile_GitHub
@@ -55,6 +55,7 @@ pipeline {
WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
+ WORKDIR_REL_JDK17_HADOOP3_CHECK = 'yetus-jdk17-hadoop3-check'
ASF_NIGHTLIES = 'https://nightlies.apache.org'
ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
ASF_NIGHTLIES_BASE = "${ASF_NIGHTLIES_BASE_ORI.replaceAll(' ', '%20')}"
@@ -470,6 +471,145 @@ pipeline {
}
}
}
+ stage ('yetus jdk17 hadoop3 checks') {
+ agent {
+ node {
+ label 'hbase'
+ }
+ }
+ environment {
+ // customized per parallel stage
+ PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-17'
+ WORKDIR_REL = "${WORKDIR_REL_JDK17_HADOOP3_CHECK}"
+ // identical for all parallel stages
+ WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
+ YETUSDIR = "${WORKDIR}/${YETUS_REL}"
+ SOURCEDIR = "${WORKDIR}/${SRC_REL}"
+ PATCHDIR = "${WORKDIR}/${PATCH_REL}"
+ BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
+ DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
+ YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+ SKIP_ERRORPRONE = true
+ }
+ when {
+ allOf {
+ // this will return true if the pipeline is building a change request, such as a GitHub pull request.
+ changeRequest()
+ expression { env.CHANGE_TARGET in ['master', 'branch-3'] }
+ }
+ }
+ steps {
+ dir("${SOURCEDIR}") {
+ checkout scm
+ }
+ dir("${YETUSDIR}") {
+ checkout([
+ $class : 'GitSCM',
+ branches : [[name: "${YETUS_VERSION}"]],
+ userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
+ )
+ }
+ dir("${WORKDIR}") {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'apache-hbase-at-github.com',
+ passwordVariable: 'GITHUB_PASSWORD',
+ usernameVariable: 'GITHUB_USER'
+ )]) {
+ script {
+ def ret = sh(
+ label: 'test-patch',
+ returnStatus: true,
+ script: '''#!/bin/bash -e
+ hostname -a ; pwd ; ls -la
+ printenv 2>&1 | sort
+ echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
+ "${YETUS_DRIVER}"
+ '''
+ )
+ if (ret != 0) {
+ // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
+ // test output. See HBASE-26339 for more details.
+ currentBuild.result = 'UNSTABLE'
+ }
+ }
+ }
+ }
+ }
+ post {
+ always {
+ junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
+ allowEmptyResults: true, skipPublishingChecks: true
+ sh label: 'zip surefire reports', script: '''#!/bin/bash -e
+ if [ -d "${PATCHDIR}/archiver" ]; then
+ count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
+ '''
+ sshPublisher(publishers: [
+ sshPublisherDesc(configName: 'Nightlies',
+ transfers: [
+ sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
+ sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
+ )
+ ]
+ )
+ ])
+ // remove the big test logs zip file, store the nightlies url in test_logs.txt
+ sh '''#!/bin/bash -e
+ if [ -f "${PATCHDIR}/test_logs.zip" ]; then
+ echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
+ rm -rf "${PATCHDIR}/test_logs.zip"
+ python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
+ else
+ echo "No test_logs.zip, skipping"
+ fi
+ '''
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
+ archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "${WORKDIR_REL}/${PATCH_REL}",
+ reportFiles: 'report.html',
+ reportName: 'PR JDK17 Hadoop3 Check Report'
+ ]
+ }
+ // Jenkins pipeline jobs fill slaves on PRs without this :(
+ cleanup() {
+ script {
+ sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
+ # See YETUS-764
+ if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+ echo "test-patch process appears to still be running: killing"
+ kill `cat "${PATCHDIR}/pidfile.txt"` || true
+ sleep 10
+ fi
+ if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+ echo "test-patch container appears to still be running: killing"
+ docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
+ fi
+ # See HADOOP-13951
+ chmod -R u+rxw "${WORKSPACE}"
+ '''
+ dir ("${WORKDIR}") {
+ deleteDir()
+ }
+ }
+ }
+ }
+ }
}
}
}
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 1f841d1d867..dcd84c89c21 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -110,6 +110,13 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \
echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c -
+FROM base_image AS openjdk17_download_image
+ENV OPENJDK17_URL 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'
+ENV OPENJDK17_SHA256 'a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378'
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \
+ echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c -
+
##
# build the final image
#
@@ -158,6 +165,14 @@ RUN mkdir -p /usr/lib/jvm && \
ln -s /usr/lib/jvm/java-11-adoptopenjdk /usr/lib/jvm/java-11 && \
rm /tmp/adoptopenjdk11.tar.gz
+# hadolint ignore=DL3010
+COPY --from=openjdk17_download_image /tmp/adoptopenjdk17.tar.gz /tmp/adoptopenjdk17.tar.gz
+RUN mkdir -p /usr/lib/jvm && \
+ tar xzf /tmp/adoptopenjdk17.tar.gz -C /usr/lib/jvm && \
+ ln -s "/usr/lib/jvm/$(basename "$(tar -tf /tmp/adoptopenjdk17.tar.gz | head -n1)")" /usr/lib/jvm/java-17-adoptopenjdk && \
+ ln -s /usr/lib/jvm/java-17-adoptopenjdk /usr/lib/jvm/java-17 && \
+ rm /tmp/adoptopenjdk17.tar.gz
+
# configure default environment for Yetus. Yetus in dockermode seems to require
# these values to be specified here; the various --foo-path flags do not
# propigate as expected, while these are honored.