You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2016/09/17 21:16:59 UTC
hive git commit: HIVE-14734: Detect ptest profile and submit to
ptest-server from jenkins-execute-build.sh (Sergio Pena,
reviewed by Siddarth Seth)
Repository: hive
Updated Branches:
refs/heads/master 05e251036 -> c90eed20d
HIVE-14734: Detect ptest profile and submit to ptest-server from jenkins-execute-build.sh (Sergio Pena, reviewed by Siddarth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c90eed20
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c90eed20
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c90eed20
Branch: refs/heads/master
Commit: c90eed20d84cf1c86dd6dd6a5dc3f56e0e48d4ff
Parents: 05e2510
Author: Sergio Pena <se...@cloudera.com>
Authored: Fri Sep 16 23:18:31 2016 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Sat Sep 17 16:16:39 2016 -0500
----------------------------------------------------------------------
dev-support/jenkins-common.sh | 63 ++++++++++++
dev-support/jenkins-execute-build.sh | 157 ++++++++++++++++++++----------
2 files changed, 166 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c90eed20/dev-support/jenkins-common.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-common.sh b/dev-support/jenkins-common.sh
index 214ca1c..0467d11 100644
--- a/dev-support/jenkins-common.sh
+++ b/dev-support/jenkins-common.sh
@@ -13,6 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+JIRA_ROOT_URL="https://issues.apache.org"
+
fail() {
echo "$@" 1>&2
exit 1
@@ -48,6 +51,66 @@ get_branch_name() {
return 0
}
+# Gets the attachment identifier of a JIRA attachment file
+get_attachment_id() {
+ local jira_attachment_url="$1"
+
+ basename $(dirname $jira_attachment_url)
+}
+
+# Fetchs JIRA information, and save it in a file specified as a parameter
+initialize_jira_info() {
+ local jira_issue="$1"
+ local output_file="$2"
+
+ curl -s -S --location --retry 3 "${JIRA_ROOT_URL}/jira/browse/${jira_issue}" > $output_file
+}
+
+# Checks if a JIRA is in 'Patch Available' status
+is_patch_available() {
+ grep -q "Patch Available" $1
+}
+
+# Checks if a JIRA has 'NO PRECOMMIT TESTS' enabled
+is_check_no_precommit_tests_set() {
+ grep -q "NO PRECOMMIT TESTS" $1
+}
+
+# Gets the URL for the JIRA patch attached
+get_jira_patch_url() {
+ grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' $1 | grep -v -e 'htm[l]*$' | sort | tail -1 | \
+ grep -o '/jira/secure/attachment/[0-9]*/[^"]*'
+}
+
+# Checks if the patch was already tested
+is_patch_already_tested() {
+ local attachment_id="$1"
+ local test_handle="$2"
+ local jira_info_file="$3"
+
+ grep -q "ATTACHMENT ID: $attachment_id $test_handle" $jira_info_file
+}
+
+# Gets the branch profile attached to the patch
+get_branch_profile() {
+ local jira_patch_url="$1"
+ local branch_name=`get_branch_name $(basename $jira_patch_url)`
+
+ if [ -n "$branch_name" ]; then
+ shopt -s nocasematch
+ if [[ $branch_name =~ (mr1|mr2)$ ]]; then
+ echo $branch_name
+ else
+ echo ${branch_name}-mr2
+ fi
+ fi
+}
+
+# Checks if a JIRA patch has 'CLEAR LIBRARY CACHE' enabled
+is_clear_cache_set() {
+ grep -q "CLEAR LIBRARY CACHE" $1
+}
+
# Parses the JIRA/patch to find relavent information.
# Exports two variables of import:
# * BUILD_PROFILE - the profile which the ptest server understands
http://git-wip-us.apache.org/repos/asf/hive/blob/c90eed20/dev-support/jenkins-execute-build.sh
----------------------------------------------------------------------
diff --git a/dev-support/jenkins-execute-build.sh b/dev-support/jenkins-execute-build.sh
index 3b41b0b..a9935e1 100644
--- a/dev-support/jenkins-execute-build.sh
+++ b/dev-support/jenkins-execute-build.sh
@@ -13,67 +13,116 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
set -e
-. jenkins-common.sh
-test -n "$BRANCH" || fail "BRANCH must be specified"
-test -n "$API_ENDPOINT" || fail "API_ENDPOINT must be specified"
-test -n "$LOG_ENDPOINT" || fail "LOG_ENDPOINT must be specified"
-test -n "$API_PASSWORD" || fail "API_PASSWORD must be specified"
-if [[ -n "$ISSUE_NUM" ]]
-then
- export JIRA_NAME="HIVE-${ISSUE_NUM}"
-fi
-export ROOT=$PWD
-export JIRA_ROOT_URL="https://issues.apache.org"
-export BUILD_TAG="${BUILD_TAG##jenkins-}"
-if [[ -n "$JIRA_NAME" ]]
-then
- echo $JIRA_NAME
-fi
set -x
-env
-if [[ -n "$JIRA_NAME" ]]
-then
- process_jira
-fi
+. jenkins-common.sh
-profile=$BUILD_PROFILE
-if [[ -z "$profile" ]]
-then
- profile=$DEFAULT_BUILD_PROFILE
-fi
-if [[ -z "$profile" ]]
-then
- fail "Could not find build profile"
-fi
+# Build the ptest framework locally
+build_ptest_client() {
+ [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined."
+
+ test -d $PTEST_BUILD_DIR || mkdir -p $PTEST_BUILD_DIR
+ cd $PTEST_BUILD_DIR && rm -rf hive
+
+ git clone --depth 1 https://github.com/apache/hive.git
+ cd hive/testutils/ptest2
+ mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$MVN_REPO_LOCAL
+}
+
+# Call the ptest server to run all tests
+call_ptest_server() {
+ [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined."
+
+ read -s -p "JIRA password: " JIRA_PASSWORD
+
+ build_ptest_client || return 1
+
+ local PTEST_CLASSPATH="$PTEST_BUILD_DIR/hive/testutils/ptest2/target/hive-ptest-1.0-classes.jar:$PTEST_BUILD_DIR/hive/testutils/ptest2/target/lib/*"
+
+ java -cp "$PTEST_CLASSPATH" org.apache.hive.ptest.api.client.PTestClient --command testStart \
+ --outputDir "$PTEST_BUILD_DIR/hive/testutils/ptest2/target" --password "$JIRA_PASSWORD" "$@"
+}
+
+# Unpack all test results
+unpack_test_results() {
+ [ -z "$PTEST_BUILD_DIR" ] && echoerr "Error: Cannot build ptest: PTEST_BUILD_DIR is not defined."
+
+ cd "$PTEST_BUILD_DIR/hive/testutils/ptest2/target"
+ if [[ -f test-results.tar.gz ]]; then
+ rm -rf $PTEST_BUILD_DIR/test-results/
+ tar zxf test-results.tar.gz -C $PTEST_BUILD_DIR
+ fi
+}
+
+# Check required arguments
+test -n "$TEST_HANDLE" || fail "TEST_HANDLE must be specified and cannot be empty."
+test -n "$PTEST_API_ENDPOINT" || fail "PTEST_API_ENDPOINT must be specified and cannot be empty."
+test -n "$PTEST_LOG_ENDPOINT" || fail "PTEST_LOG_ENDPOINT must be specified and cannot be empty."
-test -d hive/build/ || mkdir -p hive/build/
-cd hive/build/
-rm -rf hive
-git clone --depth 1 https://github.com/apache/hive.git
-cd hive/testutils/ptest2
+# WORKSPACE is an environment variable created by Jenkins, and it is the directory where the build is executed.
+# If not set, then default to $HOME
+MVN_REPO_LOCAL=${WORKSPACE:-$HOME}/.m2/repository
-mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$WORKSPACE/.m2
-set +e
+# Directory where to build the ptest framework
+PTEST_BUILD_DIR="$PWD/hive/build"
+
+# Default profile in case a patch does not have a profile assigned, or
+# if this script is executed without JIRA_ISSUE defined
+DEFAULT_BUILD_PROFILE="master-mr2"
+
+# Optional parameters that might be passed to the ptest client command
optionalArgs=()
-if [[ -n "$JIRA_NAME" ]]
-then
- optionalArgs=(--patch "${JIRA_ROOT_URL}${PATCH_URL}" --jira "$JIRA_NAME")
+
+if [ -n "$JIRA_ISSUE" ]; then
+ JIRA_INFO_FILE=`mktemp`
+ trap "rm -f $JIRA_INFO_FILE" EXIT
+
+ initialize_jira_info $JIRA_ISSUE $JIRA_INFO_FILE || fail "Error: Cannot initialize JIRA information."
+
+ if ! is_patch_available $JIRA_INFO_FILE; then
+ fail "$JIRA_ISSUE is not 'Patch Available'."
+ fi
+
+ if is_check_no_precommit_tests_set $JIRA_INFO_FILE; then
+ fail "$JIRA_ISSUE has tag NO PRECOMMIT TESTS"
+ fi
+
+ JIRA_PATCH_URL=`get_jira_patch_url $JIRA_INFO_FILE`
+ if [ -z "$JIRA_PATCH_URL" ]; then
+ fail "Unable to find attachment for $JIRA_ISSUE"
+ fi
+
+ attachment_id=`get_attachment_id $JIRA_PATCH_URL`
+ if is_patch_already_tested "$attachment_id" "$TEST_HANDLE" "$JIRA_INFO_FILE"; then
+ fail "attachment $attachment_id is already tested for $JIRA_ISSUE"
+ fi
+
+ BUILD_PROFILE=`get_branch_profile $JIRA_PATCH_URL $JIRA_INFO_FILE`
+ if [ -z "$BUILD_PROFILE" ]; then
+ BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
+ fi
+
+ if is_clear_cache_set $JIRA_INFO_FILE; then
+ optionalArgs+=(--clearLibraryCache)
+ fi
+
+ optionalArgs+=(--patch "${JIRA_ROOT_URL}${JIRA_PATCH_URL}" --jira "$JIRA_ISSUE")
+
+ echo "ISSUE: $JIRA_ISSUE PROFILE: $BUILD_PROFILE"
+else
+ # If not JIRA is specified, then use a default profile
+ BUILD_PROFILE="$DEFAULT_BUILD_PROFILE"
+
+ echo "ISSUE: unspecified PROFILE: $BUILD_PROFILE"
fi
-java -cp "target/hive-ptest-1.0-classes.jar:target/lib/*" org.apache.hive.ptest.api.client.PTestClient --endpoint "$API_ENDPOINT" \
- --logsEndpoint "$LOG_ENDPOINT" \
- --command testStart \
- --profile $profile \
- --password $API_PASSWORD \
- --outputDir target/ \
- --testHandle "$BUILD_TAG" \
- ${optionalArgs[@]} ${BUILD_OPTS} "$@"
+
+call_ptest_server --testHandle "$TEST_HANDLE" --endpoint "$PTEST_API_ENDPOINT" --logsEndpoint "$PTEST_LOG_ENDPOINT" \
+ --profile "$BUILD_PROFILE" ${optionalArgs[@]} "$@"
+
ret=$?
-cd target/
-if [[ -f test-results.tar.gz ]]
-then
- rm -rf $ROOT/hive/build/test-results/
- tar zxf test-results.tar.gz -C $ROOT/hive/build/
-fi
+
+unpack_test_results
+
exit $ret