You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2018/02/28 06:00:57 UTC
[1/5] hbase git commit: HBASE-19614 Use ArrayDeque as Queue instead
of LinkedList in CompoundBloomFilterWriter [Forced Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-15151 ce032aa8b -> 7e0b5988f (forced update)
HBASE-19614 Use ArrayDeque as Queue instead of LinkedList in CompoundBloomFilterWriter
Signed-off-by: Apekshit Sharma <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f01c90bc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f01c90bc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f01c90bc
Branch: refs/heads/HBASE-15151
Commit: f01c90bc0e3b44fbc2c500a631cd49202d2cb975
Parents: 0fa5d69
Author: BELUGA BEHR <da...@gmail.com>
Authored: Tue Feb 27 11:49:30 2018 -0800
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Tue Feb 27 11:50:52 2018 -0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f01c90bc/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java
index 0b58b21..d63a824 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CompoundBloomFilterWriter.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.io.hfile;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.LinkedList;
+import java.util.ArrayDeque;
import java.util.Queue;
import org.apache.hadoop.hbase.Cell;
@@ -71,7 +71,7 @@ public class CompoundBloomFilterWriter extends CompoundBloomFilterBase
BloomFilterChunk chunk;
}
- private Queue<ReadyChunk> readyChunks = new LinkedList<>();
+ private Queue<ReadyChunk> readyChunks = new ArrayDeque<>();
/** The first key in the current Bloom filter chunk. */
private byte[] firstKeyInChunk = null;
[2/5] hbase git commit: HBASE-20100 TestEnableTableProcedure flakey
Posted by bu...@apache.org.
HBASE-20100 TestEnableTableProcedure flakey
Allow OPEN as a possible state when update region transition state.
Usually state is OPENING but if crash before finish step is completed,
on replay, master may have read that the state is OPEN from meta table
and so will think it open... When we replay the procedure finish, allow
that the region is already OPEN.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ba063abd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ba063abd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ba063abd
Branch: refs/heads/HBASE-15151
Commit: ba063abd2f4b0aaa0622a8665c4ba96ed31eafb8
Parents: f01c90b
Author: Michael Stack <st...@apache.org>
Authored: Tue Feb 27 12:06:48 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Tue Feb 27 14:51:21 2018 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/assignment/AssignProcedure.java | 2 +-
.../org/apache/hadoop/hbase/master/assignment/RegionStates.java | 2 ++
.../hadoop/hbase/master/assignment/RegionTransitionProcedure.java | 1 +
3 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba063abd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
index 65eafe7..fd31553 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
@@ -208,7 +208,7 @@ public class AssignProcedure extends RegionTransitionProcedure {
}
}
}
- LOG.info("Start " + this + "; " + regionNode.toShortString() +
+ LOG.info("Starting " + this + "; " + regionNode.toShortString() +
"; forceNewPlan=" + this.forceNewPlan +
", retain=" + retain);
env.getAssignmentManager().queueAssign(regionNode);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba063abd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
index fa94495..adeba67 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStates.java
@@ -60,6 +60,8 @@ public class RegionStates {
private static final Logger LOG = LoggerFactory.getLogger(RegionStates.class);
protected static final State[] STATES_EXPECTED_ON_OPEN = new State[] {
+ State.OPEN, // State may already be OPEN if we died after receiving the OPEN from regionserver
+ // but before complete finish of AssignProcedure. HBASE-20100.
State.OFFLINE, State.CLOSED, // disable/offline
State.SPLITTING, State.SPLIT, // ServerCrashProcedure
State.OPENING, State.FAILED_OPEN, // already in-progress (retrying)
http://git-wip-us.apache.org/repos/asf/hbase/blob/ba063abd/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
index 8277dbe..7ce7454 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
@@ -317,6 +317,7 @@ public abstract class RegionTransitionProcedure
case REGION_TRANSITION_FINISH:
// 3. wait assignment response. completion/failure
+ LOG.debug("Finishing {}; {}", this, regionNode.toShortString());
finishTransition(env, regionNode);
am.removeRegionInTransition(regionNode, this);
return null;
[3/5] hbase git commit: HBASE-18467 report nightly results to devs
via jira
Posted by bu...@apache.org.
HBASE-18467 report nightly results to devs via jira
- rely on parallel pipeline to ensure all stages always run
- define non-CPS jira commenting function
- comment on jiras in the changeset with summary and links
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/235a3a62
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/235a3a62
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/235a3a62
Branch: refs/heads/HBASE-15151
Commit: 235a3a62f254c9946cfb6ec977e7149dbff8a3a3
Parents: ba063ab
Author: Sean Busbey <bu...@apache.org>
Authored: Wed Aug 9 00:48:46 2017 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Feb 27 23:57:47 2018 -0600
----------------------------------------------------------------------
dev-support/Jenkinsfile | 567 +++++++++++++---------
dev-support/hbase_nightly_source-artifact.sh | 1 -
2 files changed, 344 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/235a3a62/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 201783b..17b7085 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -34,6 +34,12 @@ pipeline {
// where we check out to across stages
BASEDIR = "${env.WORKSPACE}/component"
YETUS_RELEASE = '0.7.0'
+ // where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
+ OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
+ OUTPUT_DIR_RELATIVE_JDK7 = 'output-jdk7'
+ OUTPUT_DIR_RELATIVE_HADOOP2 = 'output-jdk8-hadoop2'
+ OUTPUT_DIR_RELATIVE_HADOOP3 = 'output-jdk8-hadoop3'
+
PROJECT = 'hbase'
PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
// This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
@@ -111,252 +117,367 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
stash name: 'yetus', includes: "yetus-*/*,yetus-*/**/*,tools/personality.sh"
}
}
- stage ('yetus general check') {
- environment {
- // TODO does hadoopcheck need to be jdk specific?
- // Should be things that work with multijdk
- TESTS = 'all,-unit,-findbugs'
- // on branches that don't support jdk7, this will already be JAVA_HOME, so we'll end up not
- // doing multijdk there.
- MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
- OUTPUT_DIR_RELATIVE = "output-general"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
- }
- steps {
- unstash 'yetus'
- sh '''#!/usr/bin/env bash
- rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
- rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
- "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ stage ('health checks') {
+ parallel {
+ stage ('yetus general check') {
+ environment {
+ // TODO does hadoopcheck need to be jdk specific?
+ // Should be things that work with multijdk
+ TESTS = 'all,-unit,-findbugs'
+ // on branches that don't support jdk7, this will already be JAVA_HOME, so we'll end up not
+ // doing multijdk there.
+ MULTIJDK = '/usr/lib/jvm/java-8-openjdk-amd64'
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_GENERAL}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_GENERAL}"
+ }
+ steps {
+ unstash 'yetus'
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
'''
- // TODO should this be a download from master, similar to how the personality is?
- sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
- }
- post {
- always {
- // Has to be relative to WORKSPACE.
- archive "${env.OUTPUT_DIR_RELATIVE}/*"
- archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
- publishHTML target: [
- allowMissing: true,
- keepAll: true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE
- reportDir: "${env.OUTPUT_DIR_RELATIVE}",
- reportFiles: 'console-report.html',
- reportName: 'General Nightly Build Report'
- ]
+ // TODO roll this into the hbase_nightly_yetus script
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}/commentfile}"
+ declare -i status=0
+ if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
+ echo '(/) {color:green}+1 general checks{color}' >> "${OUTPUT_DIR}/commentfile"
+ else
+ echo '(x) {color:red}-1 general checks{color}' >> "${OUTPUT_DIR}/commentfile"
+ status=1
+ fi
+ echo "-- For more information [see general report|${BUILD_URL}/General_Nightly_Build_Report/]" >> "${OUTPUT_DIR}/commentfile"
+ exit "${status}"
+ '''
+ }
+ post {
+ always {
+ // Has to be relative to WORKSPACE.
+ archive "${env.OUTPUT_DIR_RELATIVE}/*"
+ archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing: true,
+ keepAll: true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE
+ reportDir: "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles: 'console-report.html',
+ reportName: 'General Nightly Build Report'
+ ]
+ }
+ }
}
- }
- }
- stage ('yetus jdk7 checks') {
- when {
- branch 'branch-1*'
- }
- environment {
- TESTS = 'mvninstall,compile,javac,unit,htmlout'
- OUTPUT_DIR_RELATIVE = "output-jdk7"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
- // On branches where we do jdk7 checks, jdk7 will be JAVA_HOME already.
- }
- steps {
- unstash 'yetus'
- sh '''#!/usr/bin/env bash
- rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
- rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
- "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ stage ('yetus jdk7 checks') {
+ when {
+ branch 'branch-1*'
+ }
+ environment {
+ TESTS = 'mvninstall,compile,javac,unit,htmlout'
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_JDK7}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_JDK7}"
+ // On branches where we do jdk7 checks, jdk7 will be JAVA_HOME already.
+ }
+ steps {
+ unstash 'yetus'
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
'''
- sh """#!/usr/bin/env bash
- # for branch-1.1 we don't do jdk8 findbugs, so do it here
- if [ "${env.BRANCH_NAME}" == "branch-1.1" ]; then
- TESTS+=",findbugs"
- fi
- "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
- """
- }
- post {
- always {
- junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
- // zip surefire reports.
- sh '''#!/bin/bash -e
- if [ -d "${OUTPUT_DIR}/archiver" ]; then
- count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
- if [[ 0 -ne ${count} ]]; then
- echo "zipping ${count} archived files"
- zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ sh '''#!/usr/bin/env bash
+ # for branch-1.1 we don't do jdk8 findbugs, so do it here
+ if [ "${BRANCH_NAME}" == "branch-1.1" ]; then
+ TESTS+=",findbugs"
+ fi
+ rm -rf "${OUTPUT_DIR}/commentfile}"
+ declare -i status=0
+ if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
+ echo '(/) {color:green}+1 jdk7 checks{color}' >> "${OUTPUT_DIR}/commentfile"
else
- echo "No archived files, skipping compressing."
+ echo '(x) {color:red}-1 jdk7 checks{color}' >> "${OUTPUT_DIR}/commentfile"
+ status=1
fi
- else
- echo "No archiver directory, skipping compressing."
- fi
+ echo "-- For more information [see jdk7 report|${BUILD_URL}/JDK7_Nightly_Build_Report/]" >> "${OUTPUT_DIR}/commentfile"
+ exit "${status}"
+ '''
+ }
+ post {
+ always {
+ junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
+ // zip surefire reports.
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/archiver" ]; then
+ count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
'''
- // Has to be relative to WORKSPACE.
- archive "${env.OUTPUT_DIR_RELATIVE}/*"
- archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
- publishHTML target: [
- allowMissing : true,
- keepAll : true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE.
- reportDir : "${env.OUTPUT_DIR_RELATIVE}",
- reportFiles : 'console-report.html',
- reportName : 'JDK7 Nightly Build Report'
- ]
- }
- }
- }
- stage ('yetus jdk8 hadoop2 checks') {
- when {
- not {
- branch 'branch-1.1*'
+ // Has to be relative to WORKSPACE.
+ archive "${env.OUTPUT_DIR_RELATIVE}/*"
+ archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing : true,
+ keepAll : true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE.
+ reportDir : "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles : 'console-report.html',
+ reportName : 'JDK7 Nightly Build Report'
+ ]
+ }
+ }
}
- }
- environment {
- TESTS = 'mvninstall,compile,javac,unit,htmlout'
- OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop2"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
- // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
- // and is needed on branches that do both jdk7 and jdk8
- SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
- }
- steps {
- unstash 'yetus'
- sh '''#!/usr/bin/env bash
- rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
- rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
- "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ stage ('yetus jdk8 hadoop2 checks') {
+ when {
+ not {
+ branch 'branch-1.1*'
+ }
+ }
+ environment {
+ TESTS = 'mvninstall,compile,javac,unit,htmlout'
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
+ // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
+ // and is needed on branches that do both jdk7 and jdk8
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+ }
+ steps {
+ unstash 'yetus'
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
'''
- sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
- }
- post {
- always {
- junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
- // zip surefire reports.
- sh '''#!/bin/bash -e
- if [ -d "${OUTPUT_DIR}/archiver" ]; then
- count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
- if [[ 0 -ne ${count} ]]; then
- echo "zipping ${count} archived files"
- zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}/commentfile}"
+ declare -i status=0
+ if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
+ echo '(/) {color:green}+1 jdk8 hadoop2 checks{color}' >> "${OUTPUT_DIR}/commentfile"
else
- echo "No archived files, skipping compressing."
+ echo '(x) {color:red}-1 jdk8 hadoop2 checks{color}' >> "${OUTPUT_DIR}/commentfile"
+ status=1
fi
- else
- echo "No archiver directory, skipping compressing."
- fi
+ echo "-- For more information [see jdk8 (hadoop2) report|${BUILD_URL}/JDK8_Nightly_Build_Report_(Hadoop2)/]" >> "${OUTPUT_DIR}/commentfile"
+ exit "${status}"
+ '''
+ }
+ post {
+ always {
+ junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
+ // zip surefire reports.
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/archiver" ]; then
+ count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
'''
- // Has to be relative to WORKSPACE.
- archive "${env.OUTPUT_DIR_RELATIVE}/*"
- archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
- publishHTML target: [
- allowMissing : true,
- keepAll : true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE.
- reportDir : "${env.OUTPUT_DIR_RELATIVE}",
- reportFiles : 'console-report.html',
- reportName : 'JDK8 Nightly Build Report (Hadoop2)'
- ]
+ // Has to be relative to WORKSPACE.
+ archive "${env.OUTPUT_DIR_RELATIVE}/*"
+ archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing : true,
+ keepAll : true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE.
+ reportDir : "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles : 'console-report.html',
+ reportName : 'JDK8 Nightly Build Report (Hadoop2)'
+ ]
+ }
+ }
}
- }
- }
- stage ('yetus jdk8 hadoop3 checks') {
- when {
- not {
- branch 'branch-1*'
- }
- }
- environment {
- // Failure in any stage fails the build and consecutive stages are not built.
- // Findbugs is part of this last yetus stage to prevent findbugs precluding hadoop3
- // tests.
- TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
- OUTPUT_DIR_RELATIVE = "output-jdk8-hadoop3"
- OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE}"
- // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
- // and is needed on branches that do both jdk7 and jdk8
- SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
- // Activates hadoop 3.0 profile in maven runs.
- HADOOP_PROFILE = '3.0'
- }
- steps {
- unstash 'yetus'
- sh '''#!/usr/bin/env bash
- rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
- rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
- "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ stage ('yetus jdk8 hadoop3 checks') {
+ when {
+ not {
+ branch 'branch-1*'
+ }
+ }
+ environment {
+ // Failure in any stage fails the build and consecutive stages are not built.
+ // Findbugs is part of this last yetus stage to prevent findbugs precluding hadoop3
+ // tests.
+ TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
+ OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
+ OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
+ // This isn't strictly needed on branches that only support jdk8, but doesn't hurt
+ // and is needed on branches that do both jdk7 and jdk8
+ SET_JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
+ // Activates hadoop 3.0 profile in maven runs.
+ HADOOP_PROFILE = '3.0'
+ }
+ steps {
+ unstash 'yetus'
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
+ rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
'''
- sh "${env.BASEDIR}/dev-support/hbase_nightly_yetus.sh"
- }
- post {
- always {
- // Not sure how two junit test reports will work. Disabling this for now.
- // junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
- // zip surefire reports.
- sh '''#!/bin/bash -e
- if [ -d "${OUTPUT_DIR}/archiver" ]; then
- count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
- if [[ 0 -ne ${count} ]]; then
- echo "zipping ${count} archived files"
- zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ sh '''#!/usr/bin/env bash
+ rm -rf "${OUTPUT_DIR}/commentfile}"
+ declare -i status=0
+ if "${BASEDIR}/dev-support/hbase_nightly_yetus.sh" ; then
+ echo '(/) {color:green}+1 jdk8 hadoop3 checks{color}' >> "${OUTPUT_DIR}/commentfile"
else
- echo "No archived files, skipping compressing."
+ echo '(x) {color:red}-1 jdk8 hadoop3 checks{color}' >> "${OUTPUT_DIR}/commentfile"
+ status=1
fi
- else
- echo "No archiver directory, skipping compressing."
- fi
+ echo "-- For more information [see jdk8 (hadoop3) report|${BUILD_URL}/JDK8_Nightly_Build_Report_(Hadoop3)/]" >> "${OUTPUT_DIR}/commentfile"
+ exit "${status}"
+ '''
+ }
+ post {
+ always {
+ // Not sure how two junit test reports will work. Disabling this for now.
+ // junit testResults: "${env.OUTPUT_DIR_RELATIVE}/**/target/**/TEST-*.xml", allowEmptyResults: true
+ // zip surefire reports.
+ sh '''#!/bin/bash -e
+ if [ -d "${OUTPUT_DIR}/archiver" ]; then
+ count=$(find "${OUTPUT_DIR}/archiver" -type f | wc -l)
+ if [[ 0 -ne ${count} ]]; then
+ echo "zipping ${count} archived files"
+ zip -q -m -r "${OUTPUT_DIR}/test_logs.zip" "${OUTPUT_DIR}/archiver"
+ else
+ echo "No archived files, skipping compressing."
+ fi
+ else
+ echo "No archiver directory, skipping compressing."
+ fi
'''
- // Has to be relative to WORKSPACE.
- archive "${env.OUTPUT_DIR_RELATIVE}/*"
- archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
- publishHTML target: [
- allowMissing : true,
- keepAll : true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE.
- reportDir : "${env.OUTPUT_DIR_RELATIVE}",
- reportFiles : 'console-report.html',
- reportName : 'JDK8 Nightly Build Report (Hadoop3)'
- ]
+ // Has to be relative to WORKSPACE.
+ archive "${env.OUTPUT_DIR_RELATIVE}/*"
+ archive "${env.OUTPUT_DIR_RELATIVE}/**/*"
+ publishHTML target: [
+ allowMissing : true,
+ keepAll : true,
+ alwaysLinkToLastBuild: true,
+ // Has to be relative to WORKSPACE.
+ reportDir : "${env.OUTPUT_DIR_RELATIVE}",
+ reportFiles : 'console-report.html',
+ reportName : 'JDK8 Nightly Build Report (Hadoop3)'
+ ]
+ }
+ }
}
- }
- }
- // This is meant to mimic what a release manager will do to create RCs.
- // See http://hbase.apache.org/book.html#maven.release
- stage ('create source tarball') {
- tools {
- maven 'Maven (latest)'
- // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
- jdk "JDK 1.8 (latest)"
- }
- steps {
- sh '''#!/bin/bash -e
- echo "Setting up directories"
- rm -rf "output-srctarball" && mkdir "output-srctarball"
- rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
- rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
- rm -rf ".m2-for-src" && mkdir ".m2-for-src"
+ // This is meant to mimic what a release manager will do to create RCs.
+ // See http://hbase.apache.org/book.html#maven.release
+ stage ('create source tarball') {
+ tools {
+ maven 'Maven (latest)'
+ // this needs to be set to the jdk that ought to be used to build releases on the branch the Jenkinsfile is stored in.
+ jdk "JDK 1.8 (latest)"
+ }
+ steps {
+ sh '''#!/bin/bash -e
+ echo "Setting up directories"
+ rm -rf "output-srctarball" && mkdir "output-srctarball"
+ rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
+ rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
+ rm -rf ".m2-for-src" && mkdir ".m2-for-src"
'''
- sh '''#!/usr/bin/env bash
- rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
- "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
+ sh '''#!/usr/bin/env bash
+ rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
+ "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
'''
- sh """#!/bin/bash -e
- ${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh \
- --intermediate-file-dir output-srctarball \
- --unpack-temp-dir unpacked_src_tarball \
- --maven-m2-initial .m2-for-repo \
- --maven-m2-src-build .m2-for-src \
- --clean-source-checkout \
- ${env.BASEDIR}
+ sh """#!/bin/bash -e
+ if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
+ --intermediate-file-dir output-srctarball \
+ --unpack-temp-dir unpacked_src_tarball \
+ --maven-m2-initial .m2-for-repo \
+ --maven-m2-src-build .m2-for-src \
+ --clean-source-checkout \
+ "${env.BASEDIR}" ; then
+ echo '(/) {color:green}+1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile
+ else
+ echo '(x) {color:red}-1 source release artifact{color}\n-- See build output for details.' >output-srctarball/commentfile
+ fi
"""
+ }
+ post {
+ always {
+ archive 'output-srctarball/*'
+ }
+ }
+ }
+ }
+ }
+ }
+ post {
+ always {
+ script {
+ try {
+ sh "printenv"
+ def results = ["${env.OUTPUT_DIR_RELATIVE_GENERAL}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_JDK7}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_HADOOP2}/commentfile",
+ "${env.OUTPUT_DIR_RELATIVE_HADOOP3}/commentfile",
+ 'output-srctarball/commentfile']
+ echo env.BRANCH_NAME
+ echo env.BUILD_URL
+ echo currentBuild.result
+ echo currentBuild.durationString
+ def comment = "Results for branch ${env.BRANCH_NAME}, done in ${currentBuild.durationString}\n"
+ comment += "\t[build ${currentBuild.displayName} on builds.a.o|${env.BUILD_URL}]:\n----\ndetails (if available):\n\n"
+ if (currentBuild.result == "SUCCESS") {
+ comment += "(/) *{color:green}+1 overall{color}*\n\n"
+ } else {
+ comment += "(x) *{color:red}-1 overall{color}*\n"
+ // Ideally get the committer our of the change and @ mention them in the per-jira comment
+ comment += " Committer, please check your recent inclusion of a patch for this issue.\n\n"
+ }
+ echo ""
+ echo "[DEBUG] trying to aggregate step-wise results"
+ comment += results.collect { fileExists(file: it) ? readFile(file: it) : "" }.join("\n\n")
+ echo "[INFO] Comment:"
+ echo comment
+ echo ""
+ echo "[INFO] There are ${currentBuild.changeSets.size()} change sets."
+ getJirasToComment(currentBuild).each { currentIssue ->
+ jiraComment issueKey: currentIssue, body: comment
+ }
+ } catch (Exception exception) {
+ echo "Got exception: ${exception}"
+ echo " ${exception.getStackTrace()}"
+ }
}
- post {
- always {
- archive 'output-srctarball/*'
+ }
+ }
+}
+import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper
+@NonCPS
+List<String> getJirasToComment(RunWrapper thisBuild) {
+ def seenJiras = []
+ thisBuild.changeSets.each { cs ->
+ cs.getItems().each { change ->
+ CharSequence msg = change.msg
+ echo "change: ${change}"
+ echo " ${msg}"
+ echo " ${change.commitId}"
+ echo " ${change.author}"
+ echo ""
+ msg.eachMatch("HBASE-[0-9]+") { currentIssue ->
+ echo "[DEBUG] found jira key: ${currentIssue}"
+ if (currentIssue in seenJiras) {
+ echo "[DEBUG] already commented on ${currentIssue}."
+ } else {
+ echo "[INFO] commenting on ${currentIssue}."
+ seenJiras << currentIssue
}
}
}
}
+ return seenJiras
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/235a3a62/dev-support/hbase_nightly_source-artifact.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
index 375d121..b334fa5 100755
--- a/dev-support/hbase_nightly_source-artifact.sh
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -180,5 +180,4 @@ else
echo "Building a binary tarball from the source tarball failed. see srtarball_install.log for details."
exit 1
fi
-
# TODO check the layout of the binary artifact we just made.
[4/5] hbase git commit: HBASE-18467 temporarily mute posting to jira
Posted by bu...@apache.org.
HBASE-18467 temporarily mute posting to jira
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1a761e2a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1a761e2a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1a761e2a
Branch: refs/heads/HBASE-15151
Commit: 1a761e2ab3e1f7208a9c95da5bcbcf7dfa825cd8
Parents: 235a3a6
Author: Sean Busbey <bu...@apache.org>
Authored: Sun Feb 25 00:59:52 2018 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Tue Feb 27 23:57:59 2018 -0600
----------------------------------------------------------------------
dev-support/Jenkinsfile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a761e2a/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 17b7085..366806f 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -446,7 +446,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
echo ""
echo "[INFO] There are ${currentBuild.changeSets.size()} change sets."
getJirasToComment(currentBuild).each { currentIssue ->
- jiraComment issueKey: currentIssue, body: comment
+// jiraComment issueKey: currentIssue, body: comment
+ echo "jiraComment issueKey: ${currentIssue}, body: ${comment}"
}
} catch (Exception exception) {
echo "Got exception: ${exception}"
[5/5] hbase git commit: HBASE-15151 ensure findbugs check runs on all
branches.
Posted by bu...@apache.org.
HBASE-15151 ensure findbugs check runs on all branches.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7e0b5988
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7e0b5988
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7e0b5988
Branch: refs/heads/HBASE-15151
Commit: 7e0b5988fb6a816e68c32fd489efd6608e369b3f
Parents: 1a761e2
Author: Sean Busbey <bu...@apache.org>
Authored: Sun Feb 25 00:35:45 2018 -0600
Committer: Sean Busbey <bu...@apache.org>
Committed: Wed Feb 28 00:00:15 2018 -0600
----------------------------------------------------------------------
dev-support/Jenkinsfile | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7e0b5988/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 366806f..692d62d 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -241,7 +241,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
- TESTS = 'mvninstall,compile,javac,unit,htmlout'
+ TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP2}"
// This isn't strictly needed on branches that only support jdk8, but doesn't hurt
@@ -307,10 +307,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
}
}
environment {
- // Failure in any stage fails the build and consecutive stages are not built.
- // Findbugs is part of this last yetus stage to prevent findbugs precluding hadoop3
- // tests.
- TESTS = 'mvninstall,compile,javac,unit,findbugs,htmlout'
+ TESTS = 'mvninstall,compile,javac,unit,htmlout'
OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_HADOOP3}"
// This isn't strictly needed on branches that only support jdk8, but doesn't hurt