You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2017/12/22 13:45:04 UTC
[06/26] hbase git commit: HBASE-19570 Add hadoop3 tests to Nightly
master/branch-2 runs.
HBASE-19570 Add hadoop3 tests to Nightly master/branch-2 runs.
Jenkins fails the whole build immediately if any stage fails. Hadoop2 tests run before Hadoop3 tests.
So Hadoop3 tests will run only if hadoop2 tests pass.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a4272a9f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a4272a9f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a4272a9f
Branch: refs/heads/HBASE-19397
Commit: a4272a9f08f1005d1d3b2511dfbe8eca577f0d49
Parents: bcaf2fd
Author: Apekshit Sharma <ap...@apache.org>
Authored: Thu Dec 21 11:20:40 2017 -0800
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Thu Dec 21 17:44:04 2017 -0800
----------------------------------------------------------------------
dev-support/Jenkinsfile | 73 ++++++++++++++++++++++++++++++---
dev-support/hbase-personality.sh | 8 ++++
dev-support/hbase_nightly_yetus.sh | 5 +++
3 files changed, 80 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a4272a9f/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 744e9b0..dcef649 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -35,7 +35,7 @@ pipeline {
BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.6.0'
PROJECT = 'hbase'
- PROJET_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
+ PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
WHITESPACE_IGNORE_LIST = '.*/generated/.*'
@@ -102,7 +102,7 @@ fi
dir ("${env.TOOLS}") {
sh """#!/usr/bin/env bash
echo "Downloading Project personality."
-curl -L -o personality.sh "${env.PROJET_PERSONALITY}"
+curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
"""
}
stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh"
@@ -203,15 +203,15 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}"
}
}
}
- stage ('yetus jdk8 checks') {
+ stage ('yetus jdk8 hadoop2 checks') {
when {
not {
branch 'branch-1.1*'
}
}
environment {
- TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
- OUTPUT_DIR_RELATIVE = "output-jdk8"
+ TESTS = 'mvninstall,compile,javac,unit,htmlout'
+ OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop2"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
// This isn't strictly needed on branches that only support jdk8, but doesn't hurt
// and is needed on branches that do both jdk7 and jdk8
@@ -253,7 +253,68 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}"
// Has to be relative to WORKSPACE.
reportDir : "${env.OUTPUT_DIR_RELATIVE}",
reportFiles : 'console-report.html',
- reportName : 'JDK8 Nightly Build Report'
+ reportName : 'JDK8 Nightly Build Report (Hadoop2)'
+ ]
+ }
+ }
+ }
+ stage ('yetus jdk8 hadoop3 checks') {
+ when {
+ not {
+ branch 'branch-1*'
+ }
+ }
+ environment {
+ // Failure in any stage fails the build and consecutive stages are not built.
+ // Findbugs is part of this last yetus stage to prevent findbugs precluding hadoop3
+ // tests.
+ TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
+ OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop3"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
+ // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
+ // and is needed on branches that do both jdk7 and jdk8
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+ // Activates hadoop 3.0 profile in maven runs.
+ HADOOP_PROFILE = '3.0'
+ }
+ steps {
+ unstash 'yetus'
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+'''
+ sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
+ }
+ post {
+ always {
+ // Not sure how two junit test reports will work. Disabling this for now.
+ // junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
+ // zip surefire reports.
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/archiver" ]; then
+ count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
+'''
+ // Has to be relative to WORKSPACE.
+ archive "${env.OUTPUT_DIR_RELATIVE}/*"
+ archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing : true,
+ keepAll : true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE.
+ reportDir : "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles : 'console-report.html',
+ reportName : 'JDK8 Nightly Build Report (Hadoop3)'
]
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a4272a9f/dev-support/hbase-personality.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh
index cf5eacd..e24930d 100755
--- a/dev-support/hbase-personality.sh
+++ b/dev-support/hbase-personality.sh
@@ -81,6 +81,9 @@ function personality_parse_args
--include-tests-url=*)
INCLUDE_TESTS_URL=${i#*=}
;;
+ --hadoop-profile=*)
+ HADOOP_PROFILE=${i#*=}
+ ;;
esac
done
}
@@ -103,6 +106,10 @@ function personality_modules
extra="-DHBasePatchProcess"
+ if [[ -n "${HADOOP_PROFILE}" ]]; then
+ extra="${extra} -Dhadoop.profile=${HADOOP_PROFILE}"
+ fi
+
# BUILDMODE value is 'full' when there is no patch to be tested, and we are running checks on
# full source code instead. In this case, do full compiles, tests, etc instead of per
# module.
@@ -115,6 +122,7 @@ function personality_modules
fi
if [[ ${testtype} == mvninstall ]]; then
+ # shellcheck disable=SC2086
personality_enqueue_module . ${extra}
return
fi
http://git-wip-us.apache.org/repos/asf/hbase/blob/a4272a9f/dev-support/hbase_nightly_yetus.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index d37292b..e1175d2 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -76,6 +76,11 @@ if [[ -n "${INCLUDE_TESTS_URL}" && "${BRANCH_NAME}" == "master" ]]; then
YETUS_ARGS=("--include-tests-url=${INCLUDE_TESTS_URL}" "${YETUS_ARGS[@]}")
fi
+# For testing with specific hadoop version. Activates corresponding profile in maven runs.
+if [[ -n "${HADOOP_PROFILE}" ]]; then
+ YETUS_ARGS=("--hadoop-profile=${HADOOP_PROFILE}" "${YETUS_ARGS[@]}")
+fi
+
if [[ true == "${DEBUG}" ]]; then
YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}")
fi