You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2019/08/07 01:23:56 UTC

[couchdb] 02/04: Refactor using sequential stages, in workspace

This is an automated email from the ASF dual-hosted git repository.

kocolosk pushed a commit to branch jenkins-junit-reports-and-jenkins-env
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 9bc9d22f87f6bd9d044f859f888175358427bddc
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Tue Aug 6 16:49:52 2019 -0400

    Refactor using sequential stages, in workspace
    
    This work moves the builds back into the workspace, using a separate
    sub-directory per platform to avoid clashes between builds caused by
    JENKINS-57454. It also breaks out the steps into a pair of sequential
    stages within each each parallel stage of the build, which gives us
    better visibility into the progress of the build, and also sets us up
    to capture test results and expose them directly via Jenkins UI for
    faster problem determination.
---
 Jenkinsfile | 249 ++++++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 177 insertions(+), 72 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3c49b76..b2deb8e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -14,35 +14,33 @@
 // the License.
 
 // DRYing out the Jenkinsfile...
-build_script = '''
-mkdir -p ${COUCHDB_IO_LOG_DIR}
 
-echo
-echo "Build CouchDB from tarball & test"
-builddir=$(mktemp -d)
-cd ${builddir}
+build_and_test = '''
+mkdir -p ${COUCHDB_IO_LOG_DIR}
+rm -rf build
+mkdir build
+cd build
 tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
 cd apache-couchdb-*
 ./configure --with-curl
 make check || (build-aux/logfile-uploader.py && false)
+'''
 
-echo
-echo "Build CouchDB packages"
-cd ${builddir}
+make_packages = '''
 git clone https://github.com/apache/couchdb-pkg
+rm -rf couchdb
 mkdir couchdb
 cp ${WORKSPACE}/apache-couchdb-*.tar.gz couchdb
 tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz -C couchdb
 cd couchdb-pkg
 make ${platform} PLATFORM=${platform}
+'''
 
-echo
-echo "Cleanup & save for posterity"
+cleanup_and_save = '''
 rm -rf ${WORKSPACE}/pkgs/${platform}
 mkdir -p ${WORKSPACE}/pkgs/${platform}
-mv ../rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
-mv ../couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
-rm -rf ${builddir} ${COUCHDB_IO_LOG_DIR}
+mv ${WORKSPACE}/rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
+mv ${WORKSPACE}/couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
 '''
 
 pipeline {
@@ -113,11 +111,7 @@ pipeline {
     // https://issues.jenkins-ci.org/browse/JENKINS-47962
     // https://issues.jenkins-ci.org/browse/JENKINS-48050
 
-    // The builddir stuff is to prevent all the builds from live syncing
-    // their build results to each other during the build, which ACTUALLY
-    // HAPPENS. Ugh.
-
-    stage('make check') {
+    stage('Test and Package') {
 
       parallel {
 
@@ -138,18 +132,22 @@ pipeline {
                 mkdir -p $COUCHDB_IO_LOG_DIR
 
                 # Build CouchDB from tarball & test
-                builddir=$(mktemp -d)
-                cd $builddir
+                mkdir build
+                cd build
                 tar -xf $WORKSPACE/apache-couchdb-*.tar.gz
                 cd apache-couchdb-*
                 ./configure --with-curl
                 gmake check || (build-aux/logfile-uploader.py && false)
 
                 # No package build for FreeBSD at this time
-                rm -rf $builddir $COUCHDB_IO_LOG_DIR
               '''
             } // withEnv
           } // steps
+          post {
+            cleanup {
+              sh 'rm -rf $COUCHDB_IO_LOG_DIR'
+            }
+          } // post
         } // stage FreeBSD
 
         stage('CentOS 6') {
@@ -158,6 +156,8 @@ pipeline {
               image 'couchdbdev/centos-6-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              // this keeps builds landing on the same host from clashing with each other
+              customWorkspace pwd() + '/centos6'
             }
           }
           options {
@@ -167,14 +167,28 @@ pipeline {
           environment {
             platform = 'centos6'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -185,6 +199,7 @@ pipeline {
               image 'couchdbdev/centos-7-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/centos7'
             }
           }
           options {
@@ -194,14 +209,29 @@ pipeline {
           environment {
             platform = 'centos7'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                unstash 'tarball'
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -212,6 +242,7 @@ pipeline {
               image 'couchdbdev/ubuntu-xenial-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/xenial'
             }
           }
           options {
@@ -221,14 +252,28 @@ pipeline {
           environment {
             platform = 'xenial'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -239,6 +284,7 @@ pipeline {
               image 'couchdbdev/ubuntu-bionic-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/bionic'
             }
           }
           options {
@@ -248,14 +294,28 @@ pipeline {
           environment {
             platform = 'bionic'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -266,6 +326,7 @@ pipeline {
               image 'couchdbdev/debian-jessie-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/jessie'
             }
           }
           options {
@@ -275,14 +336,28 @@ pipeline {
           environment {
             platform = 'jessie'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -293,6 +368,7 @@ pipeline {
               image 'couchdbdev/debian-stretch-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/stretch'
             }
           }
           options {
@@ -302,14 +378,28 @@ pipeline {
           environment {
             platform = 'stretch'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -320,6 +410,7 @@ pipeline {
               image 'couchdbdev/aarch64-debian-stretch-erlang-20.3.8.20:latest'
               alwaysPull true
               label 'arm'
+              customWorkspace pwd() + '/arm'
             }
           }
           options {
@@ -329,14 +420,28 @@ pipeline {
           environment {
             platform = 'aarch64-debian-stretch'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -442,7 +547,7 @@ pipeline {
         body: "Boo, we failed. ${env.RUN_DISPLAY_URL}"
     }
     cleanup {
-      sh 'rm -rf ${WORKSPACE}/*'
+      sh 'rm -rf ${COUCHDB_IO_LOG_DIR}'
     }
   }