You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2020/02/22 00:50:37 UTC
[hbase] branch HBASE-23876/jdk11-nightly-master created (now
b059ca9)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a change to branch HBASE-23876/jdk11-nightly-master
in repository https://gitbox.apache.org/repos/asf/hbase.git.
at b059ca9 HBASE-23876 Add JDK11 compilation and unit test support to nightly job
This branch includes the following new commits:
new b059ca9 HBASE-23876 Add JDK11 compilation and unit test support to nightly job
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[hbase] 01/01: HBASE-23876 Add JDK11 compilation and unit test
support to nightly job
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch HBASE-23876/jdk11-nightly-master
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b059ca966f626f3abec17629fcbebb01c135d386
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Thu Feb 20 16:14:23 2020 -0800
HBASE-23876 Add JDK11 compilation and unit test support to nightly job
Builds on the Dockerfile changes provided by HBASE-23767.
---
dev-support/Jenkinsfile | 140 ++++++++++++++++++++++++++++++++++++++----------
1 file changed, 112 insertions(+), 28 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 2eb0947..69405c2 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -35,8 +35,9 @@ pipeline {
// where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
OUTPUT_DIR_RELATIVE_JDK7 = 'output-jdk7'
- OUTPUT_DIR_RELATIVE_HADOOP2 = 'output-jdk8-hadoop2'
- OUTPUT_DIR_RELATIVE_HADOOP3 = 'output-jdk8-hadoop3'
+ OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
+ OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3'
+ OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3'
PROJECT = 'hbase'
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
@@ -49,6 +50,8 @@ pipeline {
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes"
+ SHALLOW_CHECKS = 'all,-unit,-findbugs' // run by the 'yetus general check'
+ DEEP_CHECKS = 'maven,mvninstall,compile,javac,unit,htmlout' // run by 'yetus jdkX HadoopY checks'
}
parameters {
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
@@ -177,8 +180,9 @@ pipeline {
// we skip some due to e.g. branch-specific JDK or Hadoop support
stash name: 'general-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_GENERAL}/doesn't-match"
stash name: 'jdk7-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK7}/doesn't-match"
- stash name: 'hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_HADOOP2}/doesn't-match"
- stash name: 'hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_HADOOP3}/doesn't-match"
+ stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match"
+ stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match"
+ stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match"
stash name: 'srctarball-result', allowEmpty: true, includes: "output-srctarball/doesn't-match"
}
}
@@ -194,10 +198,8 @@ pipeline {
BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
- TESTS = 'all,-unit,-findbugs'
- // on branches that don't support jdk7, this will already be JAVA_HOME, so we'll end up not
- // doing multijdk there.
- MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
+ TESTS = "${env.SHALLOW_CHECKS}"
+ MULTIJDK = '/usr/lib/jvm/java-8,/usr/lib/jvm/java-11'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
}
@@ -264,10 +266,10 @@ pipeline {
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
- TESTS = 'maven,mvninstall,compile,javac,unit,htmlout'
+ TESTS = "${env.DEEP_CHECKS}"
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
- // On branches where we do jdk7 checks, jdk7 will be JAVA_HOME already.
+ SET_JAVA_HOME = "/usr/lib/jvm/java-7"
}
steps {
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
@@ -342,12 +344,10 @@ pipeline {
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
- TESTS = 'maven,mvninstall,compile,javac,unit,findbugs,htmlout'
- OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
- // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
- // and is needed on branches that do both jdk7 and jdk8
- SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+ TESTS = "${env.DEEP_CHECKS}"
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8'
}
steps {
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
@@ -383,7 +383,7 @@ pipeline {
}
post {
always {
- stash name: 'hadoop2-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ stash name: 'jdk8-hadoop2-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
// zip surefire reports.
sh '''#!/bin/bash -e
@@ -427,12 +427,10 @@ pipeline {
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
- TESTS = 'maven,mvninstall,compile,javac,unit,htmlout'
- OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
- // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
- // and is needed on branches that do both jdk7 and jdk8
- SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+ TESTS = "${env.DEEP_CHECKS}"
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8'
// Activates hadoop 3.0 profile in maven runs.
HADOOP_PROFILE = '3.0'
}
@@ -470,7 +468,7 @@ pipeline {
}
post {
always {
- stash name: 'hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ stash name: 'jdk8-hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
// zip surefire reports.
sh '''#!/bin/bash -e
@@ -501,8 +499,92 @@ pipeline {
}
}
}
+ stage ('yetus jdk11 hadoop3 checks') {
+ agent {
+ node {
+ label 'Hadoop'
+ }
+ }
+ when {
+ branch 'master'
+ }
+ environment {
+ BASEDIR = "${env.WORKSPACE}/component"
+ TESTS = "${env.DEEP_CHECKS}"
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}"
+ SET_JAVA_HOME = "/usr/lib/jvm/java-11"
+ // Activates hadoop 3.0 profile in maven runs.
+ HADOOP_PROFILE = '3.0'
+ }
+ steps {
+ // Must do prior to anything else, since if one of them timesout we'll stash the commentfile
+ sh '''#!/usr/bin/env bash
+ set -e
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ echo '(x) {color:red}-1 jdk11 hadoop3 checks{color}' >"${OUTPUT_DIR}/commentfile"
+ echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
+'''
+ unstash 'yetus'
+ dir('component') {
+ checkout scm
+ }
+ sh '''#!/usr/bin/env bash
+ set -e
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
+'''
+ sh '''#!/usr/bin/env bash
+ set -e
+ declare -i status=0
+ if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
+ echo '(/) {color:green}+1 jdk11 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
+ else
+ echo '(x) {color:red}-1 jdk11 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
+ status=1
+ fi
+ echo "-- For more information [see jdk11 report|${BUILD_URL}/JDK11_Nightly_Build_Report/]" >> "${OUTPUT_DIR}/commentfile"
+ exit "${status}"
+ '''
+ }
+ post {
+ always {
+ stash name: 'jdk11-hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
+ // zip surefire reports.
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/archiver" ]; then
+ count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
+'''
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
+ archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing : true,
+ keepAll : true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE.
+ reportDir : "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles : 'console-report.html',
+ reportName : 'JDK11 Nightly Build Report'
+ ]
+ }
+ }
+ }
// This is meant to mimic what a release manager will do to create RCs.
// See http://hbase.apache.org/book.html#maven.release
+ // TODO (HBASE-23870): replace this with invocation of the release tool
stage ('packaging and integration') {
tools {
maven 'Maven (latest)'
@@ -636,14 +718,16 @@ pipeline {
try {
unstash 'general-result'
unstash 'jdk7-result'
- unstash 'hadoop2-result'
- unstash 'hadoop3-result'
+ unstash 'jdk8-hadoop2-result'
+ unstash 'jdk8-hadoop3-result'
+ unstash 'jdk11-hadoop3-result'
unstash 'srctarball-result'
sh "printenv"
def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile",
- "${env.OUTPUT_DIR_RELATIVE_HADOOP2}/commentfile",
- "${env.OUTPUT_DIR_RELATIVE_HADOOP3}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile",
'output-srctarball/commentfile',
'output-integration/commentfile']
echo env.BRANCH_NAME