svn commit: r1634687 - in /hive/trunk/dev-support: jenkins-common.sh jenkins-execute-build.sh jenkins-submit-build.sh

[br...@apache.org]

Author: brock
Date: Mon Oct 27 21:09:19 2014
New Revision: 1634687

URL: http://svn.apache.org/r1634687
Log:
HIVE-8608 - Move jenkins scripts to source control (Brock reviewed by Szehon)

Added:
    hive/trunk/dev-support/jenkins-common.sh
    hive/trunk/dev-support/jenkins-execute-build.sh
    hive/trunk/dev-support/jenkins-submit-build.sh

Added: hive/trunk/dev-support/jenkins-common.sh
URL: http://svn.apache.org/viewvc/hive/trunk/dev-support/jenkins-common.sh?rev=1634687&view=auto
==============================================================================
--- hive/trunk/dev-support/jenkins-common.sh (added)
+++ hive/trunk/dev-support/jenkins-common.sh Mon Oct 27 21:09:19 2014
@@ -0,0 +1,92 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+fail() {
+  echo "$@" 1>&2
+  exit 1
+}
+
+# Parses the JIRA/patch to find relavent information.
+# Exports two variables of import:
+# * BUILD_PROFILE - the profile which the ptest server understands
+# * BUILD_OPTS - additional test options to be sent to ptest cli
+process_jira() {
+  test -n "$BRANCH" || fail "BRANCH must be specified"
+  test -n "$JIRA_ROOT_URL" || fail "JIRA_ROOT_URL must be specified"
+  test -n "$JIRA_NAME" || fail "API_PASSWORD must be specified"
+  JIRA_TEXT=$(mktemp)
+  trap "rm -f $JIRA_TEXT" EXIT
+  curl -s -S --location --retry 3 "${JIRA_ROOT_URL}/jira/browse/${JIRA_NAME}" > $JIRA_TEXT
+  if [[ "${CHECK_NO_PRECOMMIT_TESTS}" == "true" ]] && grep -q "NO PRECOMMIT TESTS" $JIRA_TEXT
+  then
+    fail "Test $JIRA_NAME has tag NO PRECOMMIT TESTS"
+  fi
+  # ensure the patch is actually in the correct state
+  if ! grep -q 'Patch Available' $JIRA_TEXT
+  then
+    fail "$JIRA_NAME is not \"Patch Available\". Exiting."
+  fi
+  # pull attachments from JIRA (hack stolen from hadoop since rest api doesn't show attachments)
+  PATCH_URL=$(grep -o '"/jira/secure/attachment/[0-9]*/[^"]*' $JIRA_TEXT | \
+    grep -v -e 'htm[l]*$' | sort | tail -1 | \
+    grep -o '/jira/secure/attachment/[0-9]*/[^"]*')
+  if [[ -z "$PATCH_URL" ]]
+  then
+    fail "Unable to find attachment for $JIRA_NAME"
+  fi
+  # ensure attachment has not already been tested
+  ATTACHMENT_ID=$(basename $(dirname $PATCH_URL))
+  if grep -q "ATTACHMENT ID: $ATTACHMENT_ID" $JIRA_TEXT
+  then
+    fail "Attachment $ATTACHMENT_ID is already tested for $JIRA_NAME"
+  fi
+  # validate the patch name, parse branch if needed
+  shopt -s nocasematch
+  PATCH_NAME=$(basename $PATCH_URL)
+  # Test examples:
+  # HIVE-123.patch HIVE-123.1.patch HIVE-123.D123.patch HIVE-123.D123.1.patch HIVE-123-tez.patch HIVE-123.1-tez.patch
+  # HIVE-XXXX.patch, HIVE-XXXX.XX.patch  HIVE-XXXX.XX-branch.patch HIVE-XXXX-branch.patch
+  if [[ $PATCH_NAME =~ ^HIVE-[0-9]+(\.[0-9]+)?(-[a-z0-9-]+)?\.(patch|patch.\txt)$ ]]
+  then
+    if [[ -n "${BASH_REMATCH[2]}" ]]
+    then
+      BRANCH=${BASH_REMATCH[2]#*-}
+    else
+      echo "Assuming branch $BRANCH"
+    fi
+  # HIVE-XXXX.DXXXX.patch or HIVE-XXXX.DXXXX.XX.patch
+  elif [[ $PATCH_NAME =~ ^(HIVE-[0-9]+\.)?D[0-9]+(\.[0-9]+)?\.(patch|patch.\txt)$ ]]
+  then
+    echo "Assuming branch $BRANCH"
+  else
+    fail "Patch $PATCH_NAME does not appear to be a patch"
+  fi
+  shopt -u nocasematch
+  # append mr2 if needed
+  if [[ $BRANCH =~ (mr1|mr2)$ ]]
+  then
+    profile=$BRANCH
+  else
+    profile=${BRANCH}-mr2
+  fi
+  export BUILD_PROFILE=$profile
+  build_opts=""
+  if grep -q "CLEAR LIBRARY CACHE" $JIRA_TEXT
+  then
+    echo "Clearing library cache before starting test"
+    build_opts="--clearLibraryCache"
+  fi
+  export BUILD_OPTS=$build_opts
+}

Added: hive/trunk/dev-support/jenkins-execute-build.sh
URL: http://svn.apache.org/viewvc/hive/trunk/dev-support/jenkins-execute-build.sh?rev=1634687&view=auto
==============================================================================
--- hive/trunk/dev-support/jenkins-execute-build.sh (added)
+++ hive/trunk/dev-support/jenkins-execute-build.sh Mon Oct 27 21:09:19 2014
@@ -0,0 +1,65 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+. jenkins-common.sh
+test -n "$BRANCH" || fail "BRANCH must be specified"
+test -n "$API_ENDPOINT" || fail "API_ENDPOINT must be specified"
+test -n "$LOG_ENDPOINT" || fail "LOG_ENDPOINT must be specified"
+test -n "$API_PASSWORD" || fail "API_PASSWORD must be specified"
+export JIRA_NAME="HIVE-${ISSUE_NUM}"
+export ROOT=$PWD
+export JIRA_ROOT_URL="https://issues.apache.org"
+export BUILD_TAG="${BUILD_TAG##jenkins-}"
+echo $JIRA_NAME
+set -x
+env
+
+process_jira
+
+test -d hive/build/ || mkdir -p hive/build/
+cd hive/build/
+rm -rf ptest2
+svn co http://svn.apache.org/repos/asf/hive/trunk/testutils/ptest2/ ptest2
+cd ptest2
+
+# sanity check the profile
+case "$BUILD_PROFILE" in
+  trunk-mr1);;
+  trunk-mr2);;
+  *)
+  echo "Unknown profile '$BUILD_PROFILE'"
+  exit 1
+  ;;
+esac
+mvn clean package -DskipTests -Drat.numUnapprovedLicenses=1000 -Dmaven.repo.local=$WORKSPACE/.m2
+set +e
+java -cp "target/hive-ptest-1.0-classes.jar:target/lib/*" org.apache.hive.ptest.api.client.PTestClient --endpoint "$API_ENDPOINT" \
+  --logsEndpoint "$LOG_ENDPOINT" \
+  --command testStart \
+  --profile $profile \
+  --password $API_PASSWORD \
+  --outputDir target/ \
+  --testHandle "$BUILD_TAG" \
+  --patch "${JIRA_ROOT_URL}${PATCH_URL}" \
+  --jira "$JIRA_NAME" ${BUILD_OPTS} "$@"
+ret=$?
+cd target/
+if [[ -f test-results.tar.gz ]]
+then
+  rm -rf $ROOT/hive/build/test-results/
+  tar zxf test-results.tar.gz -C $ROOT/hive/build/
+fi
+exit $ret

Added: hive/trunk/dev-support/jenkins-submit-build.sh
URL: http://svn.apache.org/viewvc/hive/trunk/dev-support/jenkins-submit-build.sh?rev=1634687&view=auto
==============================================================================
--- hive/trunk/dev-support/jenkins-submit-build.sh (added)
+++ hive/trunk/dev-support/jenkins-submit-build.sh Mon Oct 27 21:09:19 2014
@@ -0,0 +1,43 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -e
+. jenkins-common.sh
+export JIRA_NAME="HIVE-${ISSUE_NUM}"
+export JIRA_ROOT_URL="https://issues.apache.org"
+export BRANCH=trunk
+echo $JIRA_NAME
+
+process_jira
+
+# sanity check the profile
+case "$BUILD_PROFILE" in
+  trunk-mr1|trunk-mr2)
+   test -n "$TRUNK_URL" || fail "TRUNK_URL must be specified"
+   url="$TRUNK_URL&ISSUE_NUM=$ISSUE_NUM"
+   curl -v -i "$url"
+   exit 0
+  ;;
+  spark-mr2|spark2-mr2)
+   test -n "$SPARK_URL" || fail "SPARK_URL must be specified"
+   url="$SPARK_URL&ISSUE_NUM=$ISSUE_NUM"
+   curl -v -i "$url"
+   exit 0
+  ;;
+  *)
+  echo "Unknown profile '$BUILD_PROFILE'"
+  exit 1
+  ;;
+esac