You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2018/04/14 04:18:22 UTC
[1/4] hbase git commit: HBASE-20335 ensure each stage of the nightly
job gathers machine information.
Repository: hbase
Updated Branches:
refs/heads/branch-2 8cd1201af -> 1d133c005
refs/heads/branch-2.0 7704d3bb1 -> 4de3d7dee
HBASE-20335 ensure each stage of the nightly job gathers machine information.
* fix archiving for src tarball stage's machine info
* stop nightly wrapper desroying the output dir.
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dcd20e9c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dcd20e9c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dcd20e9c
Branch: refs/heads/branch-2
Commit: dcd20e9cefa5d0a70728b22d259747b795091ae5
Parents: 8cd1201
Author: Sean Busbey <bu...@apache.org>
Authored: Wed Apr 11 10:38:12 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 22:45:19 2018 -0500
----------------------------------------------------------------------
dev-support/Jenkinsfile | 11 +++++++++++
dev-support/hbase_nightly_yetus.sh | 7 +++++--
2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcd20e9c/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 6acd4b2..56d53d8 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -150,6 +150,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
// TODO roll this into the hbase_nightly_yetus script
sh '''#!/usr/bin/env bash
@@ -210,6 +212,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
sh '''#!/usr/bin/env bash
set -e
@@ -283,6 +287,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
sh '''#!/usr/bin/env bash
set -e
@@ -363,6 +369,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
sh '''#!/usr/bin/env bash
set -e
@@ -437,6 +445,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
set -e
rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
+ echo "got the following saved stats in 'output-srctarball/machine'"
+ ls -lh "output-srctarball/machine"
'''
sh """#!/bin/bash -e
if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
@@ -456,6 +466,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
always {
stash name: 'srctarball-result', includes: "output-srctarball/commentfile"
archive 'output-srctarball/*'
+ archive 'output-srctarball/**/*'
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dcd20e9c/dev-support/hbase_nightly_yetus.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 4e0200d..bba5f4d 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -91,8 +91,11 @@ if [[ true == "${DEBUG}" ]]; then
YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}")
fi
-rm -rf "${OUTPUT_DIR}"
-mkdir -p "${OUTPUT_DIR}"
+if [[ ! -d "${OUTPUT_DIR}" ]]; then
+ echo "[ERROR] the specified output directory must already exist: '${OUTPUT_DIR}'"
+ exit 1
+fi
+
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
[3/4] hbase git commit: HBASE-20335 ensure each stage of the nightly
job gathers machine information.
Posted by bu...@apache.org.
HBASE-20335 ensure each stage of the nightly job gathers machine information.
* fix archiving for src tarball stage's machine info
* stop nightly wrapper desroying the output dir.
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41762373
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41762373
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41762373
Branch: refs/heads/branch-2.0
Commit: 417623736ff676b151a2b37f4a8107d1ce4872df
Parents: 7704d3b
Author: Sean Busbey <bu...@apache.org>
Authored: Wed Apr 11 10:38:12 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 23:16:29 2018 -0500
----------------------------------------------------------------------
dev-support/Jenkinsfile | 11 +++++++++++
dev-support/hbase_nightly_yetus.sh | 7 +++++--
2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/41762373/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 6acd4b2..56d53d8 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -150,6 +150,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
// TODO roll this into the hbase_nightly_yetus script
sh '''#!/usr/bin/env bash
@@ -210,6 +212,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
sh '''#!/usr/bin/env bash
set -e
@@ -283,6 +287,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
sh '''#!/usr/bin/env bash
set -e
@@ -363,6 +369,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+ echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+ ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
'''
sh '''#!/usr/bin/env bash
set -e
@@ -437,6 +445,8 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
set -e
rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
"${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
+ echo "got the following saved stats in 'output-srctarball/machine'"
+ ls -lh "output-srctarball/machine"
'''
sh """#!/bin/bash -e
if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
@@ -456,6 +466,7 @@ curl -L -o personality.sh "${env.PROJECT_PERSONALITY}"
always {
stash name: 'srctarball-result', includes: "output-srctarball/commentfile"
archive 'output-srctarball/*'
+ archive 'output-srctarball/**/*'
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/41762373/dev-support/hbase_nightly_yetus.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 4e0200d..bba5f4d 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -91,8 +91,11 @@ if [[ true == "${DEBUG}" ]]; then
YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}")
fi
-rm -rf "${OUTPUT_DIR}"
-mkdir -p "${OUTPUT_DIR}"
+if [[ ! -d "${OUTPUT_DIR}" ]]; then
+ echo "[ERROR] the specified output directory must already exist: '${OUTPUT_DIR}'"
+ exit 1
+fi
+
if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
[4/4] hbase git commit: HBASE-20389 Move website building flags into
a profile.
Posted by bu...@apache.org.
HBASE-20389 Move website building flags into a profile.
Signed-off-by: Mike Drob <md...@apache.org>
Conflicts:
hbase-spark/pom.xml
pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4de3d7de
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4de3d7de
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4de3d7de
Branch: refs/heads/branch-2.0
Commit: 4de3d7deefd536025abf2ee81517e8035082274c
Parents: 4176237
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Apr 12 21:55:27 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 23:16:37 2018 -0500
----------------------------------------------------------------------
.../jenkins-scripts/generate-hbase-website.sh | 11 ++-----
pom.xml | 31 ++++++++++++++++++++
2 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/4de3d7de/dev-support/jenkins-scripts/generate-hbase-website.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh
index b6277d0..0ef9b2d 100644
--- a/dev-support/jenkins-scripts/generate-hbase-website.sh
+++ b/dev-support/jenkins-scripts/generate-hbase-website.sh
@@ -173,20 +173,13 @@ echo "Building HBase"
# But! some sunshine: because we're doing a full install before running site, we can skip all the
# compiling in the forked executions. We have to do it awkwardly because MJAVADOC-444.
if mvn \
- -DskipTests \
- -Dmaven.javadoc.skip=true \
--batch-mode \
- -Denforcer.skip=true \
- -Dcheckstyle.skip=true \
- -Dfindbugs.skip=true \
+ -Psite-install-step \
--log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" \
clean install \
&& mvn site \
--batch-mode \
- -Denforcer.skip=true \
- -Dmaven.main.skip=true \
- -Dmaven.test.skip=true \
- -DskipTests \
+ -Psite-build-step \
--log-file="${working_dir}/hbase-site-log-${CURRENT_HBASE_COMMIT}.txt"; then
echo "Successfully built site."
else
http://git-wip-us.apache.org/repos/asf/hbase/blob/4de3d7de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3fe436b..8224f64 100755
--- a/pom.xml
+++ b/pom.xml
@@ -3375,6 +3375,37 @@
</plugins>
</build>
</profile>
+ <profile>
+ <!-- Used by the website generation script on jenkins to
+ do a local install of the jars we need to run a normal
+ site build w/o forking.
+ -->
+ <id>site-install-step</id>
+ <properties>
+ <skipTests>true</skipTests>
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ <enforcer.skip>true</enforcer.skip>
+ <checkstyle.skip>true</checkstyle.skip>
+ <findbugs.skip>true</findbugs.skip>
+ <warbucks.skip>true</warbucks.skip>
+ </properties>
+ </profile>
+ <profile>
+ <!-- Used by the website generation script on jenkins to
+ mitigate the impact of unneeded build forks while building
+ our javadocs.
+ -->
+ <id>site-build-step</id>
+ <properties>
+ <skipTests>true</skipTests>
+ <enforcer.skip>true</enforcer.skip>
+ <maven.main.skip>true</maven.main.skip>
+ <maven.test.skip>true</maven.test.skip>
+ <warbucks.skip>true</warbucks.skip>
+ <protoc.skip>true</protoc.skip>
+ <remoteresources.skip>true</remoteresources.skip>
+ </properties>
+ </profile>
</profiles>
<!-- See http://jira.codehaus.org/browse/MSITE-443 why the settings need to be here and not in pluginManagement. -->
<reporting>
[2/4] hbase git commit: HBASE-20389 Move website building flags into
a profile.
Posted by bu...@apache.org.
HBASE-20389 Move website building flags into a profile.
Signed-off-by: Mike Drob <md...@apache.org>
Conflicts:
hbase-spark/pom.xml
pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1d133c00
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1d133c00
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1d133c00
Branch: refs/heads/branch-2
Commit: 1d133c005c26c01628fcc622c3095ddccb80bd6b
Parents: dcd20e9
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Apr 12 21:55:27 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 22:50:55 2018 -0500
----------------------------------------------------------------------
.../jenkins-scripts/generate-hbase-website.sh | 11 ++-----
pom.xml | 31 ++++++++++++++++++++
2 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1d133c00/dev-support/jenkins-scripts/generate-hbase-website.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh
index b6277d0..0ef9b2d 100644
--- a/dev-support/jenkins-scripts/generate-hbase-website.sh
+++ b/dev-support/jenkins-scripts/generate-hbase-website.sh
@@ -173,20 +173,13 @@ echo "Building HBase"
# But! some sunshine: because we're doing a full install before running site, we can skip all the
# compiling in the forked executions. We have to do it awkwardly because MJAVADOC-444.
if mvn \
- -DskipTests \
- -Dmaven.javadoc.skip=true \
--batch-mode \
- -Denforcer.skip=true \
- -Dcheckstyle.skip=true \
- -Dfindbugs.skip=true \
+ -Psite-install-step \
--log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" \
clean install \
&& mvn site \
--batch-mode \
- -Denforcer.skip=true \
- -Dmaven.main.skip=true \
- -Dmaven.test.skip=true \
- -DskipTests \
+ -Psite-build-step \
--log-file="${working_dir}/hbase-site-log-${CURRENT_HBASE_COMMIT}.txt"; then
echo "Successfully built site."
else
http://git-wip-us.apache.org/repos/asf/hbase/blob/1d133c00/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4709004..9c78ef9 100755
--- a/pom.xml
+++ b/pom.xml
@@ -3376,6 +3376,37 @@
</plugins>
</build>
</profile>
+ <profile>
+ <!-- Used by the website generation script on jenkins to
+ do a local install of the jars we need to run a normal
+ site build w/o forking.
+ -->
+ <id>site-install-step</id>
+ <properties>
+ <skipTests>true</skipTests>
+ <maven.javadoc.skip>true</maven.javadoc.skip>
+ <enforcer.skip>true</enforcer.skip>
+ <checkstyle.skip>true</checkstyle.skip>
+ <findbugs.skip>true</findbugs.skip>
+ <warbucks.skip>true</warbucks.skip>
+ </properties>
+ </profile>
+ <profile>
+ <!-- Used by the website generation script on jenkins to
+ mitigate the impact of unneeded build forks while building
+ our javadocs.
+ -->
+ <id>site-build-step</id>
+ <properties>
+ <skipTests>true</skipTests>
+ <enforcer.skip>true</enforcer.skip>
+ <maven.main.skip>true</maven.main.skip>
+ <maven.test.skip>true</maven.test.skip>
+ <warbucks.skip>true</warbucks.skip>
+ <protoc.skip>true</protoc.skip>
+ <remoteresources.skip>true</remoteresources.skip>
+ </properties>
+ </profile>
</profiles>
<!-- See http://jira.codehaus.org/browse/MSITE-443 why the settings need to be here and not in pluginManagement. -->
<reporting>