You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2018/04/14 04:18:22 UTC

[1/4] hbase git commit: HBASE-20335 ensure each stage of the nightly job gathers machine information.

Repository: hbase
Updated Branches:
  refs/heads/branch-2 8cd1201af -> 1d133c005
  refs/heads/branch-2.0 7704d3bb1 -> 4de3d7dee


HBASE-20335 ensure each stage of the nightly job gathers machine information.

* fix archiving for src tarball stage's machine info
* stop nightly wrapper desroying the output dir.

Signed-off-by: Michael Stack <st...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dcd20e9c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dcd20e9c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dcd20e9c

Branch: refs/heads/branch-2
Commit: dcd20e9cefa5d0a70728b22d259747b795091ae5
Parents: 8cd1201
Author: Sean Busbey <bu...@apache.org>
Authored: Wed Apr 11 10:38:12 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 22:45:19 2018 -0500

----------------------------------------------------------------------
 dev-support/Jenkinsfile            | 11 +++++++++++
 dev-support/hbase_nightly_yetus.sh |  7 +++++--
 2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/dcd20e9c/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 6acd4b2..56d53d8 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -150,6 +150,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             // TODO roll this into the hbase_nightly_yetus script
             sh '''#!/usr/bin/env bash
@@ -210,6 +212,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             sh '''#!/usr/bin/env bash
               set -e
@@ -283,6 +287,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             sh '''#!/usr/bin/env bash
               set -e
@@ -363,6 +369,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             sh '''#!/usr/bin/env bash
               set -e
@@ -437,6 +445,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               set -e
               rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
+              echo "got the following saved stats in 'output-srctarball/machine'"
+              ls -lh "output-srctarball/machine"
 '''
             sh """#!/bin/bash -e
               if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
@@ -456,6 +466,7 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
             always {
               stash name: 'srctarball-result', includes: "output-srctarball/commentfile"
               archive 'output-srctarball/*'
+              archive 'output-srctarball/**/*'
             }
           }
         }

http://git-wip-us.apache.org/repos/asf/hbase/blob/dcd20e9c/dev-support/hbase_nightly_yetus.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 4e0200d..bba5f4d 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -91,8 +91,11 @@ if [[ true == "${DEBUG}" ]]; then
   YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}")
 fi
 
-rm -rf "${OUTPUT_DIR}"
-mkdir -p "${OUTPUT_DIR}"
+if [[ ! -d "${OUTPUT_DIR}" ]]; then
+  echo "[ERROR] the specified output directory must already exist: '${OUTPUT_DIR}'"
+  exit 1
+fi
+
 if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
   YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
   TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"


[3/4] hbase git commit: HBASE-20335 ensure each stage of the nightly job gathers machine information.

Posted by bu...@apache.org.
HBASE-20335 ensure each stage of the nightly job gathers machine information.

* fix archiving for src tarball stage's machine info
* stop nightly wrapper desroying the output dir.

Signed-off-by: Michael Stack <st...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41762373
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41762373
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41762373

Branch: refs/heads/branch-2.0
Commit: 417623736ff676b151a2b37f4a8107d1ce4872df
Parents: 7704d3b
Author: Sean Busbey <bu...@apache.org>
Authored: Wed Apr 11 10:38:12 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 23:16:29 2018 -0500

----------------------------------------------------------------------
 dev-support/Jenkinsfile            | 11 +++++++++++
 dev-support/hbase_nightly_yetus.sh |  7 +++++--
 2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/41762373/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 6acd4b2..56d53d8 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -150,6 +150,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             // TODO roll this into the hbase_nightly_yetus script
             sh '''#!/usr/bin/env bash
@@ -210,6 +212,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             sh '''#!/usr/bin/env bash
               set -e
@@ -283,6 +287,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             sh '''#!/usr/bin/env bash
               set -e
@@ -363,6 +369,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               rm -rf "${OUTPUT_DIR}" && mkdir "${OUTPUT_DIR}"
               rm -rf "${OUTPUT_DIR}/machine" && mkdir "${OUTPUT_DIR}/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine"
+              echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'"
+              ls -lh "${OUTPUT_DIR_RELATIVE}/machine"
 '''
             sh '''#!/usr/bin/env bash
               set -e
@@ -437,6 +445,8 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
               set -e
               rm -rf "output-srctarball/machine" && mkdir "output-srctarball/machine"
               "${BASEDIR}/dev-support/gather_machine_environment.sh" "output-srctarball/machine"
+              echo "got the following saved stats in 'output-srctarball/machine'"
+              ls -lh "output-srctarball/machine"
 '''
             sh """#!/bin/bash -e
               if "${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh" \
@@ -456,6 +466,7 @@ curl -L  -o personality.sh "${env.PROJECT_PERSONALITY}"
             always {
               stash name: 'srctarball-result', includes: "output-srctarball/commentfile"
               archive 'output-srctarball/*'
+              archive 'output-srctarball/**/*'
             }
           }
         }

http://git-wip-us.apache.org/repos/asf/hbase/blob/41762373/dev-support/hbase_nightly_yetus.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_yetus.sh b/dev-support/hbase_nightly_yetus.sh
index 4e0200d..bba5f4d 100755
--- a/dev-support/hbase_nightly_yetus.sh
+++ b/dev-support/hbase_nightly_yetus.sh
@@ -91,8 +91,11 @@ if [[ true == "${DEBUG}" ]]; then
   YETUS_ARGS=("--debug" "${YETUS_ARGS[@]}")
 fi
 
-rm -rf "${OUTPUT_DIR}"
-mkdir -p "${OUTPUT_DIR}"
+if [[ ! -d "${OUTPUT_DIR}" ]]; then
+  echo "[ERROR] the specified output directory must already exist: '${OUTPUT_DIR}'"
+  exit 1
+fi
+
 if [[ true !=  "${USE_YETUS_PRERELEASE}" ]]; then
   YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
   TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"


[4/4] hbase git commit: HBASE-20389 Move website building flags into a profile.

Posted by bu...@apache.org.
HBASE-20389 Move website building flags into a profile.

Signed-off-by: Mike Drob <md...@apache.org>

 Conflicts:
	hbase-spark/pom.xml
	pom.xml


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4de3d7de
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4de3d7de
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4de3d7de

Branch: refs/heads/branch-2.0
Commit: 4de3d7deefd536025abf2ee81517e8035082274c
Parents: 4176237
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Apr 12 21:55:27 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 23:16:37 2018 -0500

----------------------------------------------------------------------
 .../jenkins-scripts/generate-hbase-website.sh   | 11 ++-----
 pom.xml                                         | 31 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/4de3d7de/dev-support/jenkins-scripts/generate-hbase-website.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh
index b6277d0..0ef9b2d 100644
--- a/dev-support/jenkins-scripts/generate-hbase-website.sh
+++ b/dev-support/jenkins-scripts/generate-hbase-website.sh
@@ -173,20 +173,13 @@ echo "Building HBase"
 # But! some sunshine: because we're doing a full install before running site, we can skip all the
 # compiling in the forked executions. We have to do it awkwardly because MJAVADOC-444.
 if mvn \
-    -DskipTests \
-    -Dmaven.javadoc.skip=true \
     --batch-mode \
-    -Denforcer.skip=true \
-    -Dcheckstyle.skip=true \
-    -Dfindbugs.skip=true \
+    -Psite-install-step \
     --log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" \
     clean install \
   && mvn site \
     --batch-mode \
-    -Denforcer.skip=true \
-    -Dmaven.main.skip=true \
-    -Dmaven.test.skip=true \
-    -DskipTests \
+    -Psite-build-step \
     --log-file="${working_dir}/hbase-site-log-${CURRENT_HBASE_COMMIT}.txt"; then
   echo "Successfully built site."
 else

http://git-wip-us.apache.org/repos/asf/hbase/blob/4de3d7de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3fe436b..8224f64 100755
--- a/pom.xml
+++ b/pom.xml
@@ -3375,6 +3375,37 @@
             </plugins>
         </build>
     </profile>
+    <profile>
+      <!-- Used by the website generation script on jenkins to
+           do a local install of the jars we need to run a normal
+           site build w/o forking.
+        -->
+      <id>site-install-step</id>
+      <properties>
+        <skipTests>true</skipTests>
+        <maven.javadoc.skip>true</maven.javadoc.skip>
+        <enforcer.skip>true</enforcer.skip>
+        <checkstyle.skip>true</checkstyle.skip>
+        <findbugs.skip>true</findbugs.skip>
+        <warbucks.skip>true</warbucks.skip>
+      </properties>
+    </profile>
+    <profile>
+      <!-- Used by the website generation script on jenkins to
+           mitigate the impact of unneeded build forks while building
+           our javadocs.
+        -->
+      <id>site-build-step</id>
+      <properties>
+        <skipTests>true</skipTests>
+        <enforcer.skip>true</enforcer.skip>
+        <maven.main.skip>true</maven.main.skip>
+        <maven.test.skip>true</maven.test.skip>
+        <warbucks.skip>true</warbucks.skip>
+        <protoc.skip>true</protoc.skip>
+        <remoteresources.skip>true</remoteresources.skip>
+      </properties>
+    </profile>
   </profiles>
   <!-- See http://jira.codehaus.org/browse/MSITE-443 why the settings need to be here and not in pluginManagement. -->
   <reporting>


[2/4] hbase git commit: HBASE-20389 Move website building flags into a profile.

Posted by bu...@apache.org.
HBASE-20389 Move website building flags into a profile.

Signed-off-by: Mike Drob <md...@apache.org>

 Conflicts:
	hbase-spark/pom.xml
	pom.xml


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1d133c00
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1d133c00
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1d133c00

Branch: refs/heads/branch-2
Commit: 1d133c005c26c01628fcc622c3095ddccb80bd6b
Parents: dcd20e9
Author: Sean Busbey <bu...@apache.org>
Authored: Thu Apr 12 21:55:27 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Fri Apr 13 22:50:55 2018 -0500

----------------------------------------------------------------------
 .../jenkins-scripts/generate-hbase-website.sh   | 11 ++-----
 pom.xml                                         | 31 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/1d133c00/dev-support/jenkins-scripts/generate-hbase-website.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-scripts/generate-hbase-website.sh b/dev-support/jenkins-scripts/generate-hbase-website.sh
index b6277d0..0ef9b2d 100644
--- a/dev-support/jenkins-scripts/generate-hbase-website.sh
+++ b/dev-support/jenkins-scripts/generate-hbase-website.sh
@@ -173,20 +173,13 @@ echo "Building HBase"
 # But! some sunshine: because we're doing a full install before running site, we can skip all the
 # compiling in the forked executions. We have to do it awkwardly because MJAVADOC-444.
 if mvn \
-    -DskipTests \
-    -Dmaven.javadoc.skip=true \
     --batch-mode \
-    -Denforcer.skip=true \
-    -Dcheckstyle.skip=true \
-    -Dfindbugs.skip=true \
+    -Psite-install-step \
     --log-file="${working_dir}/hbase-install-log-${CURRENT_HBASE_COMMIT}.txt" \
     clean install \
   && mvn site \
     --batch-mode \
-    -Denforcer.skip=true \
-    -Dmaven.main.skip=true \
-    -Dmaven.test.skip=true \
-    -DskipTests \
+    -Psite-build-step \
     --log-file="${working_dir}/hbase-site-log-${CURRENT_HBASE_COMMIT}.txt"; then
   echo "Successfully built site."
 else

http://git-wip-us.apache.org/repos/asf/hbase/blob/1d133c00/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4709004..9c78ef9 100755
--- a/pom.xml
+++ b/pom.xml
@@ -3376,6 +3376,37 @@
             </plugins>
         </build>
     </profile>
+    <profile>
+      <!-- Used by the website generation script on jenkins to
+           do a local install of the jars we need to run a normal
+           site build w/o forking.
+        -->
+      <id>site-install-step</id>
+      <properties>
+        <skipTests>true</skipTests>
+        <maven.javadoc.skip>true</maven.javadoc.skip>
+        <enforcer.skip>true</enforcer.skip>
+        <checkstyle.skip>true</checkstyle.skip>
+        <findbugs.skip>true</findbugs.skip>
+        <warbucks.skip>true</warbucks.skip>
+      </properties>
+    </profile>
+    <profile>
+      <!-- Used by the website generation script on jenkins to
+           mitigate the impact of unneeded build forks while building
+           our javadocs.
+        -->
+      <id>site-build-step</id>
+      <properties>
+        <skipTests>true</skipTests>
+        <enforcer.skip>true</enforcer.skip>
+        <maven.main.skip>true</maven.main.skip>
+        <maven.test.skip>true</maven.test.skip>
+        <warbucks.skip>true</warbucks.skip>
+        <protoc.skip>true</protoc.skip>
+        <remoteresources.skip>true</remoteresources.skip>
+      </properties>
+    </profile>
   </profiles>
   <!-- See http://jira.codehaus.org/browse/MSITE-443 why the settings need to be here and not in pluginManagement. -->
   <reporting>