You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2017/11/02 20:36:32 UTC
[4/7] hbase git commit: HBASE-19124 Move nightly test of source
artifact creation from Jenkinsfile to a script.
HBASE-19124 Move nightly test of source artifact creation from Jenkinsfile to a script.
Signed-off-by: Apekshit Sharma <ap...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cd05007d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cd05007d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cd05007d
Branch: refs/heads/branch-1.4
Commit: cd05007da3b0a6a1ec5c642cd8cb492e4ba40cff
Parents: 59a8d1b
Author: Sean Busbey <bu...@apache.org>
Authored: Wed Nov 1 08:12:03 2017 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Thu Nov 2 15:32:54 2017 -0500
----------------------------------------------------------------------
dev-support/Jenkinsfile | 61 +++-----
dev-support/hbase_nightly_source-artifact.sh | 168 ++++++++++++++++++++++
2 files changed, 186 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/cd05007d/dev-support/Jenkinsfile
----------------------------------------------------------------------
diff --git a/dev-support/Jenkinsfile b/dev-support/Jenkinsfile
index 10438a4..dfac5b3 100644
--- a/dev-support/Jenkinsfile
+++ b/dev-support/Jenkinsfile
@@ -258,51 +258,26 @@ curl -L -o personality.sh "${env.PROJET_PERSONALITY}"
jdk "JDK 1.7 (latest)"
}
steps {
- sh 'mvn --version --offline'
- dir ("component") {
- sh '''#!/bin/bash -e
- echo "Clean..."
- mvn --batch-mode -DskipTests clean
- echo "Step 3 Build the source tarball"
- mvn -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-repo" install -DskipTests assembly:single -Dassembly.file=hbase-assembly/src/main/assembly/src.xml
-'''
- }
- dir ("unpacked_src_tarball") {
- sh '''#!/bin/bash -e
- echo "Unpack the source tarball"
- tar --strip-components=1 -xzf ../component/hbase-assembly/target/hbase-*-src.tar.gz
- echo "Building from source artifact."
- mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${WORKSPACE}/.m2-for-src" clean install assembly:single
- echo "Clean up after checking ability to build."
- mvn -DskipTests --batch-mode clean
-'''
- }
- dir ("component") {
- sh '''#!/bin/bash -e
- echo "Clean up the source checkout"
- mvn -DskipTests --batch-mode clean
- echo "Diff against source tree"
- diff --binary --recursive . ../unpacked_src_tarball >../diff_output || true
-'''
- }
- // expectation check largely based on HBASE-14952
sh '''#!/bin/bash -e
- echo "Checking against things we don't expect to include in the source tarball (git related, hbase-native-client, etc.)"
- cat >known_excluded <<END
-Only in .: .git
-Only in .: .gitattributes
-Only in .: .gitignore
-Only in .: hbase-native-client
-END
- if ! diff known_excluded diff_output >/dev/null ; then
- echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch."
- echo "The expected differences are on the < side and the current differences are on the > side."
- echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball."
- diff known_excluded diff_output
- else
- echo "Everything looks as expected."
- fi
+ echo "Setting up directories"
+ rm -rf "output-srctarball" && mkdir "output-srctarball"
+ rm -rf "unpacked_src_tarball" && mkdir "unpacked_src_tarball"
+ rm -rf ".m2-for-repo" && mkdir ".m2-for-repo"
+ rm -rf ".m2-for-src" && mkdir ".m2-for-src"
'''
+ sh """#!/bin/bash -e
+ ${env.BASEDIR}/dev-support/hbase_nightly_source-artifact.sh \
+ --intermediate-file-dir output-srctarball \
+ --unpack-temp-dir unpacked_src_tarball \
+ --maven-m2-initial .m2-for-repo \
+ --maven-m2-src-build .m2-for-src \
+ ${env.BASEDIR}
+"""
+ }
+ post {
+ always {
+ archive 'output-srctarball/*'
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/cd05007d/dev-support/hbase_nightly_source-artifact.sh
----------------------------------------------------------------------
diff --git a/dev-support/hbase_nightly_source-artifact.sh b/dev-support/hbase_nightly_source-artifact.sh
new file mode 100755
index 0000000..4e4c6a2
--- /dev/null
+++ b/dev-support/hbase_nightly_source-artifact.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] /path/to/component/checkout"
+ echo ""
+ echo " --intermediate-file-dir /path/to/use Path for writing listings and diffs. must exist."
+ echo " defaults to making a directory in /tmp."
+ echo " --unpack-temp-dir /path/to/use Path for unpacking tarball. default to"
+ echo " 'unpacked_src_tarball' in intermediate directory."
+ echo " --maven-m2-initial /path/to/use Path for maven artifacts while building in"
+ echo " component-dir."
+ echo " --maven-m2-src-build /path/to/use Path for maven artifacts while building from the"
+ echo " unpacked source tarball."
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 1 ]; then
+ usage
+fi
+
+# Get arguments
+declare component_dir
+declare unpack_dir
+declare m2_initial
+declare m2_tarbuild
+declare working_dir
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --unpack-temp-dir) shift; unpack_dir=$1; shift;;
+ --maven-m2-initial) shift; m2_initial=$1; shift;;
+ --maven-m2-src-build) shift; m2_tarbuild=$1; shift;;
+ --intermediate-file-dir) shift; working_dir=$1; shift;;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have where component checkout is.
+if [ $# -lt 1 ]; then
+ usage
+fi
+component_dir="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
+
+if [ -z "${working_dir}" ]; then
+ working_dir=/tmp
+ while [[ -e ${working_dir} ]]; do
+ working_dir=/tmp/hbase-srctarball-test-${RANDOM}.${RANDOM}
+ done
+ mkdir "${working_dir}"
+else
+ # absolutes please
+ working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
+ if [ ! -d "${working_dir}" ]; then
+ echo "passed working directory '${working_dir}' must already exist."
+ exit 1
+ fi
+fi
+
+echo "You'll find logs and temp files in ${working_dir}"
+
+if [ -z "${unpack_dir}" ]; then
+ unpack_dir="${working_dir}/unpacked_src_tarball"
+ mkdir "${unpack_dir}"
+else
+ # absolutes please
+ unpack_dir="$(cd "$(dirname "${unpack_dir}")"; pwd)/$(basename "${unpack_dir}")"
+ if [ ! -d "${unpack_dir}" ]; then
+ echo "passed directory for unpacking the source tarball '${unpack_dir}' must already exist."
+ exit 1
+ fi
+ rm -rf "${unpack_dir:?}/*"
+fi
+
+if [ -z "${m2_initial}" ]; then
+ m2_initial="${working_dir}/.m2-initial"
+ mkdir "${m2_initial}"
+else
+ # absolutes please
+ m2_initial="$(cd "$(dirname "${m2_initial}")"; pwd)/$(basename "${m2_initial}")"
+ if [ ! -d "${m2_initial}" ]; then
+ echo "passed directory for storing the initial build's maven repo '${m2_initial}' " \
+ "must already exist."
+ exit 1
+ fi
+fi
+
+if [ -z "${m2_tarbuild}" ]; then
+ m2_tarbuild="${working_dir}/.m2-tarbuild"
+ mkdir "${m2_tarbuild}"
+else
+ # absolutes please
+ m2_tarbuild="$(cd "$(dirname "${m2_tarbuild}")"; pwd)/$(basename "${m2_tarbuild}")"
+ if [ ! -d "${m2_tarbuild}" ]; then
+ echo "passed directory for storing the build from src tarball's maven repo '${m2_tarbuild}' " \
+ "must already exist."
+ exit 1
+ fi
+fi
+
+# This is meant to mimic what a release manager will do to create RCs.
+# See http://hbase.apache.org/book.html#maven.release
+
+echo "Maven details, in case our JDK doesn't match expectations:"
+mvn --version --offline | tee "${working_dir}/maven_version"
+
+echo "Do a clean building of the source artifact using code in ${component_dir}"
+cd "${component_dir}"
+echo "Clean..."
+mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_before.log" 2>&1
+echo "Step 3 Build the source tarball"
+mvn -Prelease --batch-mode -Dmaven.repo.local="${m2_initial}" install -DskipTests assembly:single \
+ -Dassembly.file=hbase-assembly/src/main/assembly/src.xml \
+ >"${working_dir}/component_build_src_tarball.log" 2>&1
+
+cd "${unpack_dir}"
+echo "Unpack the source tarball"
+tar --strip-components=1 -xzf "${component_dir}"/hbase-assembly/target/hbase-*-src.tar.gz \
+ >"${working_dir}/srctarball_unpack.log" 2>&1
+echo "Building from source artifact."
+mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
+ assembly:single >"${working_dir}/srctarball_install.log" 2>&1
+
+echo "Clean up after checking ability to build."
+mvn -DskipTests --batch-mode clean >"${working_dir}/srctarball_clean.log" 2>&1
+
+
+cd "${component_dir}"
+echo "Clean up the source checkout"
+mvn --batch-mode -DskipTests clean >"${working_dir}/component_clean_after.log" 2>&1
+echo "Diff against source tree"
+diff --binary --recursive . "${unpack_dir}" >"${working_dir}/diff_output" || true
+
+cd "${working_dir}"
+# expectation check largely based on HBASE-14952
+echo "Checking against things we don't expect to include in the source tarball (git related, hbase-native-client, etc.)"
+cat >known_excluded <<END
+Only in .: .git
+Only in .: .gitattributes
+Only in .: .gitignore
+Only in .: hbase-native-client
+END
+if ! diff known_excluded diff_output >"${working_dir}/unexpected.diff" ; then
+ echo "Any output here are unexpected differences between the source artifact we'd make for an RC and the current branch."
+ echo "The expected differences are on the < side and the current differences are on the > side."
+ echo "In a given set of differences, '.' refers to the branch in the repo and 'unpacked_src_tarball' refers to what we pulled out of the tarball."
+ diff known_excluded diff_output
+else
+ echo "Everything looks as expected."
+fi