You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2020/02/25 22:58:06 UTC
[hbase] branch HBASE-23876/jdk11-nightly-master updated (ad25217 ->
3351eba)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a change to branch HBASE-23876/jdk11-nightly-master
in repository https://gitbox.apache.org/repos/asf/hbase.git.
omit ad25217 [DO NOT MERGE] include YETUS-943
omit 3210060 HBASE-23876 Add JDK11 compilation and unit test support to nightly job
new 35bf87b HBASE-23876 Add JDK11 compilation and unit test support to nightly job
new 3351eba [DO NOT MERGE] include YETUS-943
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ad25217)
\
N -- N -- N refs/heads/HBASE-23876/jdk11-nightly-master (3351eba)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dev-support/Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
[hbase] 01/02: HBASE-23876 Add JDK11 compilation and unit test
support to nightly job
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch HBASE-23876/jdk11-nightly-master
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 35bf87b95e61177350d9f97114f11332e05bb9d8
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Thu Feb 20 16:14:23 2020 -0800
HBASE-23876 Add JDK11 compilation and unit test support to nightly job
Builds on the Dockerfile changes provided by HBASE-23767.
---
dev-support/Jenkinsfile | 141 ++++++++++++++++++++++++++++++++++++++----------
1 file changed, 113 insertions(+), 28 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 2eb0947..579d90d 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -35,8 +35,9 @@ pipeline {
// where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
OUTPUT_DIR_RELATIVE_JDK7 = 'output-jdk7'
- OUTPUT_DIR_RELATIVE_HADOOP2 = 'output-jdk8-hadoop2'
- OUTPUT_DIR_RELATIVE_HADOOP3 = 'output-jdk8-hadoop3'
+ OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
+ OUTPUT_DIR_RELATIVE_JDK8_HADOOP3 = 'output-jdk8-hadoop3'
+ OUTPUT_DIR_RELATIVE_JDK11_HADOOP3 = 'output-jdk11-hadoop3'
PROJECT = 'hbase'
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
@@ -49,6 +50,8 @@ pipeline {
// These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/excludes"
+ SHALLOW_CHECKS = 'all,-unit,-findbugs' // run by the 'yetus general check'
+ DEEP_CHECKS = 'maven,mvninstall,compile,javac,unit,htmlout' // run by 'yetus jdkX HadoopY checks'
}
parameters {
booleanParam(name: 'USE_YETUS_PRERELEASE', defaultValue: false, description: '''Check to use the current HEAD of apache/yetus rather than our configured release.
@@ -177,8 +180,9 @@ pipeline {
// we skip some due to e.g. branch-specific JDK or Hadoop support
stash name: 'general-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_GENERAL}/doesn't-match"
stash name: 'jdk7-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK7}/doesn't-match"
- stash name: 'hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_HADOOP2}/doesn't-match"
- stash name: 'hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_HADOOP3}/doesn't-match"
+ stash name: 'jdk8-hadoop2-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/doesn't-match"
+ stash name: 'jdk8-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/doesn't-match"
+ stash name: 'jdk11-hadoop3-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/doesn't-match"
stash name: 'srctarball-result', allowEmpty: true, includes: "output-srctarball/doesn't-match"
}
}
@@ -194,10 +198,9 @@ pipeline {
BASEDIR = "${env.WORKSPACE}/component"
// TODO does hadoopcheck need to be jdk specific?
// Should be things that work with multijdk
- TESTS = 'all,-unit,-findbugs'
- // on branches that don't support jdk7, this will already be JAVA_HOME, so we'll end up not
- // doing multijdk there.
- MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
+ TESTS = "${env.SHALLOW_CHECKS}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8'
+ MULTIJDK = '/usr/lib/jvm/java-8,/usr/lib/jvm/java-11'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
}
@@ -264,10 +267,10 @@ pipeline {
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
- TESTS = 'maven,mvninstall,compile,javac,unit,htmlout'
+ TESTS = "${env.DEEP_CHECKS}"
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
- // On branches where we do jdk7 checks, jdk7 will be JAVA_HOME already.
+ SET_JAVA_HOME = "/usr/lib/jvm/java-7"
}
steps {
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
@@ -342,12 +345,10 @@ pipeline {
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
- TESTS = 'maven,mvninstall,compile,javac,unit,findbugs,htmlout'
- OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
- // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
- // and is needed on branches that do both jdk7 and jdk8
- SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+ TESTS = "${env.DEEP_CHECKS}"
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8'
}
steps {
// Must do prior to anything else, since if one of them timesout we'll stash the commentfile
@@ -383,7 +384,7 @@ pipeline {
}
post {
always {
- stash name: 'hadoop2-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ stash name: 'jdk8-hadoop2-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
// zip surefire reports.
sh '''#!/bin/bash -e
@@ -427,12 +428,10 @@ pipeline {
}
environment {
BASEDIR = "${env.WORKSPACE}/component"
- TESTS = 'maven,mvninstall,compile,javac,unit,htmlout'
- OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
- // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
- // and is needed on branches that do both jdk7 and jdk8
- SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+ TESTS = "${env.DEEP_CHECKS}"
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}"
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8'
// Activates hadoop 3.0 profile in maven runs.
HADOOP_PROFILE = '3.0'
}
@@ -470,7 +469,7 @@ pipeline {
}
post {
always {
- stash name: 'hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ stash name: 'jdk8-hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
// zip surefire reports.
sh '''#!/bin/bash -e
@@ -501,8 +500,92 @@ pipeline {
}
}
}
+ stage ('yetus jdk11 hadoop3 checks') {
+ agent {
+ node {
+ label 'Hadoop'
+ }
+ }
+ when {
+ branch pattern: '.*master.*', comparator: "REGEXP"
+ }
+ environment {
+ BASEDIR = "${env.WORKSPACE}/component"
+ TESTS = "${env.DEEP_CHECKS}"
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}"
+ SET_JAVA_HOME = "/usr/lib/jvm/java-11"
+ // Activates hadoop 3.0 profile in maven runs.
+ HADOOP_PROFILE = '3.0'
+ }
+ steps {
+ // Must do prior to anything else, since if one of them timesout we'll stash the commentfile
+ sh '''#!/usr/bin/env bash
+ set -e
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ echo '(x) {color:red}-1 jdk11 hadoop3 checks{color}' >"${OUTPUT_DIR}/commentfile"
+ echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile"
+'''
+ unstash 'yetus'
+ dir('component') {
+ checkout scm
+ }
+ sh '''#!/usr/bin/env bash
+ set -e
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
+'''
+ sh '''#!/usr/bin/env bash
+ set -e
+ declare -i status=0
+ if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
+ echo '(/) {color:green}+1 jdk11 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
+ else
+ echo '(x) {color:red}-1 jdk11 hadoop3 checks{color}' > "${OUTPUT_DIR}/commentfile"
+ status=1
+ fi
+ echo "-- For more information [see jdk11 report|${BUILD_URL}/JDK11_Nightly_Build_Report/]" >> "${OUTPUT_DIR}/commentfile"
+ exit "${status}"
+ '''
+ }
+ post {
+ always {
+ stash name: 'jdk11-hadoop3-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile"
+ junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
+ // zip surefire reports.
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/archiver" ]; then
+ count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
+'''
+ // Has to be relative to WORKSPACE.
+ archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/*"
+ archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing : true,
+ keepAll : true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE.
+ reportDir : "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles : 'console-report.html',
+ reportName : 'JDK11 Nightly Build Report'
+ ]
+ }
+ }
+ }
// This is meant to mimic what a release manager will do to create RCs.
// See http://hbase.apache.org/book.html#maven.release
+ // TODO (HBASE-23870): replace this with invocation of the release tool
stage ('packaging and integration') {
tools {
maven 'Maven (latest)'
@@ -636,14 +719,16 @@ pipeline {
try {
unstash 'general-result'
unstash 'jdk7-result'
- unstash 'hadoop2-result'
- unstash 'hadoop3-result'
+ unstash 'jdk8-hadoop2-result'
+ unstash 'jdk8-hadoop3-result'
+ unstash 'jdk11-hadoop3-result'
unstash 'srctarball-result'
sh "printenv"
def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile",
"${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile",
- "${env.OUTPUT_DIR_RELATIVE_HADOOP2}/commentfile",
- "${env.OUTPUT_DIR_RELATIVE_HADOOP3}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP2}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_JDK8_HADOOP3}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_JDK11_HADOOP3}/commentfile",
'output-srctarball/commentfile',
'output-integration/commentfile']
echo env.BRANCH_NAME
[hbase] 02/02: [DO NOT MERGE] include YETUS-943
Posted by nd...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch HBASE-23876/jdk11-nightly-master
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 3351eba6a067050b8d5883bfd2b4331cb4a92db1
Author: Nick Dimiduk <nd...@apache.org>
AuthorDate: Mon Feb 24 14:50:19 2020 -0800
[DO NOT MERGE] include YETUS-943
---
dev-support/Jenkinsfile | 24 ++++--------------------
1 file changed, 4 insertions(+), 20 deletions(-)
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 579d90d..f3e3530 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -81,26 +81,10 @@ pipeline {
sh '''#!/usr/bin/env bash
set -e
echo "Ensure we have a copy of Apache Yetus."
- if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
- YETUS_DIR="${WORKSPACE}/yetus-${YETUS_RELEASE}"
- echo "Checking for Yetus ${YETUS_RELEASE} in '${YETUS_DIR}'"
- if ! "${YETUS_DIR}/bin/test-patch" --version >/dev/null 2>&1 ; then
- rm -rf "${YETUS_DIR}"
- "${WORKSPACE}/component/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
- --working-dir "${WORKSPACE}/downloads-yetus" \
- --keys 'https://www.apache.org/dist/yetus/KEYS' \
- "${WORKSPACE}/yetus-${YETUS_RELEASE}-bin.tar.gz" \
- "yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
- mv "yetus-${YETUS_RELEASE}-bin.tar.gz" yetus.tar.gz
- else
- echo "Reusing cached install of Apache Yetus version ${YETUS_RELEASE}."
- fi
- else
- YETUS_DIR="${WORKSPACE}/yetus-git"
- rm -rf "${YETUS_DIR}"
- echo "downloading from github"
- curl -L --fail https://api.github.com/repos/apache/yetus/tarball/HEAD -o yetus.tar.gz
- fi
+ YETUS_DIR="${WORKSPACE}/yetus-git"
+ rm -rf "${YETUS_DIR}"
+ echo "downloading from github"
+ curl -L --fail https://api.github.com/repos/ndimiduk/yetus/tarball/943-jdk-version -o yetus.tar.gz
if [ ! -d "${YETUS_DIR}" ]; then
echo "unpacking yetus into '${YETUS_DIR}'"
mkdir -p "${YETUS_DIR}"