You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/02/07 03:08:53 UTC
[GitHub] mrutkows closed pull request #25: Add the support to git clone all source code based on configurations
mrutkows closed pull request #25: Add the support to git clone all source code based on configurations
URL: https://github.com/apache/incubator-openwhisk-release/pull/25
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index 4147f1a..bda6e54 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,14 +5,19 @@ group: deprecated-2017Q3
services:
- docker
+env:
+ global:
+ - secure: S4eRZSGj/pfsicg50UXrTmwRurzgUrd0idQH5nKvc+O+lwAYv3SIhbsufUHseQEUSfoO0jFCyoCbWDtFMdsW3t2eGPdciViek/LaocxxG2Br539VDVXhS+NAwqiRGbCqKHCe2aNqIdo8vCvUaj/BfFJNUK4Fhpw6iBix0pvlnRkYBX7hHphU88N69gmnMOo1oZwEwaobaQeXOc/8OwH2aPD8lCmNk41IvvLFAwIhXrP4Thb+xBh4+oSlqcJyDCUouIfYFd4B1pHQehAVu0kBoUiKZHfKUyT+Q/V8hzBR3nJ/IzzSLNOYcTTYXBbL5x/SKe/LiBio4yLdPuRMTZl/xKarhShHJg3JKRvfushaMJDKJfLrbcCPUDnSCK1v0D4zZc0DceOBawhipSPkam3kNER/ImZcpTj79u/VKBrlwbVwjKTyZ4d/4/N2cP4jyvjmwW6fM4N3kM/V+S+EoGR5EiFTKAiPBzKaRi+/+VUO8ha9TsdTu+vkDRUjDFuYLSod0aj8XHUoZiL7NF8qS1PK6AFfTRpet1NM6BDOWYh15GmuZ/noyXTZ5eJv/8/ecfLMEp+DXAZ+YVI6a9/xhxE1klI88zHIoVMmhwd76jcvHVLolKa8fX9Dv5rZeOcPLoR2/jiteoApOGD0ehnkJH76gUMyBQO+q3yzRzz1VQvjOyA=
+ - secure: zqevjGa1iX3I6ZM1upNvNhEVBRq5sYpfK1pAzj6t7nTyCNvZoftTeMtDpP6eo6xAo+PatycEubwZ/pwac3V142AAgYrLXwqlojWWMQmx11BoTa3n1uJTtQyBr/5qjOLYs8VE5eDFKcXvXLQ5IDtLSZiYFGxmWQGInUNQlBaP9MQIRb2w+UgZBrp+1jRuaFs9I0+eLz3XLsyp9DeUa70lm49QiniKEktJdcUEQpQR0906k9DaRsvA3BIVsLtBwmUSg5+v8tsy13yKsPZgWNYfAjLU2DZHUQBQVj22FeSTMGOiCFSdRejhhFu9HpR7eJGoKMQGRzdTDLWZechVcVim849eVBjS6VfNBLAHOlyxIxy1Qf530610BYW2bBcaU3PhDjIcW8B+Z7EFZmYdpJRaT4CT7metLKTszy6W/splqG+3XP2XH4KmN8YPSikNiPagGP4beFIHxmCcNopHtJGhD4R8w6xqCKvNbtjq3GiUyU3ju+kA1JL0IxAttHTFF1WVTHjcXstopfXw+9HIUETm++OoKN6fxZ7Ysq7RvetkeJabL9Sqi1IsMJ3oDUdLoktTxYcE8A0gTpnLZRa2IdMJZ0ee2ZsMGieeAqfcJQ+61BMjxBTfrC9k+9gAj3zjgWsXOBMqePuQ8SwIT0H7/QgH5V/tP1wYvRUaJdJJoVoXJ2I=
+
jobs:
include:
- - stage: verify source code
- script: ./tools/travis/verify_source_code.sh
- - stage: deploy
- script: ./tools/travis/setup.sh
+ #- stage: verify source code
+ # script: ./tools/travis/verify_source_code.sh
+ #- stage: deploy
+ # script: ./tools/travis/setup.sh
- stage: package
- script: ./tools/travis/package_source_code.sh
+ script: ./tools/travis/package_source_code.sh $(dirname "$TRAVIS_BUILD_DIR") $SVN_USERNAME $SVN_PASSWORD
after_failure:
- echo "failed build"
diff --git a/tools/clean_remote_artifacts.sh b/tools/clean_remote_artifacts.sh
new file mode 100755
index 0000000..8e31d97
--- /dev/null
+++ b/tools/clean_remote_artifacts.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "Clean the remote artifacts"
+
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+source "$SCRIPTDIR/util.sh"
+
+CONFIG=$(read_file $SCRIPTDIR/config.json)
+version_key="version"
+version_major=$(json_by_key "$CONFIG" ${version_key}.major)
+version_minor=$(json_by_key "$CONFIG" ${version_key}.minor)
+
+version=$version_major-$version_minor
+REMOTE_PATH="openwhisk-$version"
+STAGE_URL=$(json_by_key "$CONFIG" "stage_url")
+CURRENT_VERSION_URL="$STAGE_URL/${REMOTE_PATH}/"
+CREDENTIALS=""
+
+SVN_USERNAME=$1
+SVN_PASSWORD=$2
+
+if [ ! -z "$SVN_USERNAME" ] && [ ! -z "$SVN_PASSWORD" ];then
+ CREDENTIALS="--username $SVN_USERNAME --password $SVN_PASSWORD --non-interactive"
+fi
+
+if [[ `wget -S --spider $CURRENT_VERSION_URL 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then
+ svn delete $CURRENT_VERSION_URL -m "Removing Apache OpenWhisk release ${version} from staging." $CREDENTIALS
+fi
+
+
+
diff --git a/tools/config.json b/tools/config.json
new file mode 100644
index 0000000..5b9e217
--- /dev/null
+++ b/tools/config.json
@@ -0,0 +1,27 @@
+{
+ "stage_url": "https://dist.apache.org/repos/dist/dev/incubator/openwhisk",
+ "version": {
+ "major": "incubator",
+ "minor": "rc1"
+ },
+ "RepoList": [
+ "incubator-openwhisk",
+ "incubator-openwhisk-wskdeploy",
+ "incubator-openwhisk-cli"
+ ],
+ "incubator_openwhisk": {
+ "hash": "a891e06",
+ "repository": "https://github.com/apache/incubator-openwhisk.git",
+ "branch": "master"
+ },
+ "incubator_openwhisk_wskdeploy": {
+ "hash": "1425456",
+ "repository": "https://github.com/apache/incubator-openwhisk-wskdeploy.git",
+ "branch": "master"
+ },
+ "incubator_openwhisk_cli": {
+ "hash": "90db37e",
+ "repository": "https://github.com/houshengbo/incubator-openwhisk-cli.git",
+ "branch": "add-zip-source"
+ }
+}
diff --git a/tools/download_source_code.sh b/tools/download_source_code.sh
new file mode 100755
index 0000000..8837139
--- /dev/null
+++ b/tools/download_source_code.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -e
+
+WORK_DIR=${1:-"$HOME"}
+OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources"
+
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+source "$SCRIPTDIR/util.sh"
+
+clone_depth=${2:-"100"}
+
+function git_clone_repo() {
+ PROJECT_NAME=$1
+ HASH=$2
+ REPO=$3
+ BRANCH=$4
+ git clone --depth $clone_depth -b $BRANCH $REPO $OPENWHISK_SOURCE_DIR/$PROJECT_NAME
+ cd $OPENWHISK_SOURCE_DIR/$PROJECT_NAME
+ git reset --hard $HASH
+ rm -rf .git
+}
+
+rm -rf $OPENWHISK_SOURCE_DIR/*
+
+CONFIG=$(read_file $SCRIPTDIR/config.json)
+repos=$(echo $(json_by_key "$CONFIG" "RepoList") | sed 's/[][]//g')
+
+for repo in $(echo $repos | sed "s/,/ /g")
+do
+ repo_name=$(echo "$repo" | sed -e 's/^"//' -e 's/"$//')
+ HASH_KEY=${repo_name//-/_}.hash
+ REPO_KEY=${repo_name//-/_}.repository
+ REPO_BRANCH=${repo_name//-/_}.branch
+ HASH=$(json_by_key "$CONFIG" $HASH_KEY)
+ REPO=$(json_by_key "$CONFIG" $REPO_KEY)
+ BRANCH=$(json_by_key "$CONFIG" $REPO_BRANCH)
+ if [ "$HASH" != "null" ]; then
+ echo "The hash for $repo_name is $HASH"
+ git_clone_repo $repo_name $HASH $REPO $BRANCH
+ fi
+done
diff --git a/tools/install_dependencies.sh b/tools/install_dependencies.sh
new file mode 100755
index 0000000..a4cbbac
--- /dev/null
+++ b/tools/install_dependencies.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -e
+
+sysOS=`uname -s`
+if [ $sysOS == "Darwin" ];then
+ echo "This is MacOS."
+ brew install jq
+elif [ $sysOS == "Linux" ];then
+ echo "This is Linux."
+ if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then
+ export DISTRO=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//)
+ else
+ export DISTRO=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1)
+ fi
+ if [ $DISTRO == "Ubuntu" ];then
+ echo "This is Ubuntu."
+ sudo apt-get install jq
+ fi
+else
+ echo "Other OS: $sysOS. Please manually install jq library."
+fi
diff --git a/tools/package_source_code.sh b/tools/package_source_code.sh
new file mode 100755
index 0000000..d97ad5d
--- /dev/null
+++ b/tools/package_source_code.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "Package the artifacts."
+
+SVN_USERNAME=$2
+SVN_PASSWORD=$3
+CREDENTIALS=""
+
+if [ ! -z "$SVN_USERNAME" ] && [ ! -z "$SVN_PASSWORD" ];then
+ CREDENTIALS="--username $SVN_USERNAME --password $SVN_PASSWORD --non-interactive"
+fi
+
+WORK_DIR=${1:-"$HOME"}
+
+OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources"
+OPENWHISK_SVN="$OPENWHISK_SOURCE_DIR/openwhisk"
+
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+source "$SCRIPTDIR/util.sh"
+
+CONFIG=$(read_file $SCRIPTDIR/config.json)
+repos=$(echo $(json_by_key "$CONFIG" "RepoList") | sed 's/[][]//g')
+version_key="version"
+version_major=$(json_by_key "$CONFIG" ${version_key}.major)
+version_minor=$(json_by_key "$CONFIG" ${version_key}.minor)
+
+version=$version_major-$version_minor
+CURRENT_VERSION_DIR="$OPENWHISK_SVN/openwhisk-$version"
+echo $version
+
+STAGE_URL=$(json_by_key "$CONFIG" "stage_url")
+echo $STAGE_URL
+
+# Create a subversion directory for openwhisk to stage all the packages
+rm -rf $OPENWHISK_SVN
+
+mkdir -p $OPENWHISK_SOURCE_DIR
+cd $OPENWHISK_SOURCE_DIR
+svn co $STAGE_URL $OPENWHISK_SVN $CREDENTIALS
+mkdir -p $CURRENT_VERSION_DIR
+rm -rf $CURRENT_VERSION_DIR/*
+
+for repo in $(echo $repos | sed "s/,/ /g")
+do
+ repo_name=$(echo "$repo" | sed -e 's/^"//' -e 's/"$//')
+ echo $repo_name
+ if [ $repo_name != "incubator-openwhisk" ]; then
+ # Only wskdeploy has both the gradle tasks implemented: cleanBuild to clean the build folder, and taredSources
+ # to package the source code.
+ cd $OPENWHISK_SOURCE_DIR/$repo_name
+ ./gradlew cleanBuild
+ ./gradlew taredSources -PprojectVersion=$version
+ # Copy all the source code packages into $CURRENT_VERSION_DIR
+ cp -a build/. $CURRENT_VERSION_DIR
+ fi
+done
+
diff --git a/tools/travis/config.json b/tools/travis/config.json
deleted file mode 100644
index e4077f0..0000000
--- a/tools/travis/config.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "RepoList": [
- "incubator-openwhisk",
- "incubator-openwhisk-catalog",
- "incubator-openwhisk-runtime-nodejs",
- "incubator-openwhisk-cli",
- "incubator-openwhisk-wskdeploy"
- ],
- "incubator_openwhisk": {
- "hash": "a891e06"
- },
- "incubator_openwhisk_catalog": {
- "hash": "ad24d4c"
- },
- "incubator_openwhisk_runtime_nodejs": {
- "hash": "1c6c5db"
- },
- "incubator_openwhisk_cli": {
- "hash": "7b78091"
- },
- "incubator_openwhisk_wskdeploy": {
- "hash": "9840db4"
- }
-}
diff --git a/tools/travis/download_source_code.sh b/tools/travis/download_source_code.sh
deleted file mode 100755
index 336746e..0000000
--- a/tools/travis/download_source_code.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-sudo apt-get install jq
-
-HOMEDIR="$(dirname "$TRAVIS_BUILD_DIR")"
-OPENWHISKDIR="$HOMEDIR/openwhisk"
-source "$TRAVIS_BUILD_DIR/tools/travis/util.sh"
-
-clone_depth=${1:-"100"}
-
-function git_clone_repo() {
- PROJECT_NAME=$1
- HASH=$2
- git clone --depth $clone_depth https://github.com/apache/$PROJECT_NAME.git $OPENWHISKDIR/$PROJECT_NAME
- cd $OPENWHISKDIR/$PROJECT_NAME
- git reset --hard $HASH
- rm -rf .git
-}
-
-rm -rf $OPENWHISKDIR/*
-
-CONFIG=$(read_file $TRAVIS_BUILD_DIR/tools/travis/config.json)
-repos=$(echo $(json_by_key "$CONFIG" "RepoList") | sed 's/[][]//g')
-
-for repo in $(echo $repos | sed "s/,/ /g")
-do
- repo_name=$(echo "$repo" | sed -e 's/^"//' -e 's/"$//')
- HASH_KEY=${repo_name//-/_}.hash
- HASH=$(json_by_key "$CONFIG" $HASH_KEY)
- if [ "$HASH" != "null" ]; then
- echo "The hash for $repo_name is $HASH"
- git_clone_repo $repo_name $HASH
- fi
-done
diff --git a/tools/travis/package_source_code.sh b/tools/travis/package_source_code.sh
index 62df67e..305c967 100755
--- a/tools/travis/package_source_code.sh
+++ b/tools/travis/package_source_code.sh
@@ -2,10 +2,24 @@
set -e
-cd $TRAVIS_BUILD_DIR
-./tools/travis/download_source_code.sh
+WORK_DIR=${1:-"$(dirname "$TRAVIS_BUILD_DIR")"}
+OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources"
-# TODO: add release note and license file if necessary
-echo "add release note and license file if necessary"
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+PARENTDIR="$(dirname "$SCRIPTDIR")"
-# TODO: package all the source code and upload them into the designated directory
+SVN_USERNAME=$2
+SVN_PASSWORD=$3
+
+"$PARENTDIR/install_dependencies.sh"
+"$PARENTDIR/download_source_code.sh" $WORK_DIR
+
+if [ "$TRAVIS_EVENT_TYPE" == "push" ] ; then
+ "$PARENTDIR/clean_remote_artifacts.sh" $SVN_USERNAME $SVN_PASSWORD
+fi
+
+"$PARENTDIR/package_source_code.sh" $WORK_DIR $SVN_USERNAME $SVN_PASSWORD
+
+if [ "$TRAVIS_EVENT_TYPE" == "push" ] ; then
+ "$PARENTDIR/upload_artifacts.sh" $WORK_DIR $SVN_USERNAME $SVN_PASSWORD
+fi
diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh
index d79c58c..1ccdaca 100755
--- a/tools/travis/setup.sh
+++ b/tools/travis/setup.sh
@@ -5,8 +5,11 @@ set -e
echo "hello openwhisk release"
# Define the parent directory of TRAVIS_BUILD_DIR(the directory of the current repository) as HOMEDIR
-HOMEDIR="$(dirname "$TRAVIS_BUILD_DIR")"
-OPENWHISKDIR="$HOMEDIR/openwhisk"
+WORK_DIR=${1:-"$(dirname "$TRAVIS_BUILD_DIR")"}
+OPENWHISKDIR="$WORK_DIR/openwhisk_sources"
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+PARENTDIR="$(dirname "$SCRIPTDIR")"
+
SUFFIX="$TRAVIS_BUILD_NUMBER"
PR_NUM="$TRAVIS_PULL_REQUEST"
PREFIX="build_testing"
@@ -19,8 +22,7 @@ cd $OPENWHISKDIR
echo "list images in setup before build"
docker images
-cd $TRAVIS_BUILD_DIR
-./tools/travis/download_source_code.sh
+"$PARENTDIR/download_source_code.sh" $WORK_DIR
cd $OPENWHISKDIR/incubator-openwhisk
./tools/travis/setup.sh
@@ -38,8 +40,8 @@ $ANSIBLE_CMD openwhisk.yml
# Build the binaries for CLI
cd $OPENWHISKDIR/incubator-openwhisk-cli
-./gradlew buildBinaries -PcrossCompileCLI=true
+#./gradlew buildBinaries -PcrossCompileCLI=true
# Build the binaries for wskdeploy
cd $OPENWHISKDIR/incubator-openwhisk-wskdeploy
-./gradlew distDocker -PcrossCompileWSKDEPLOY=true
+#./gradlew distDocker -PcrossCompileWSKDEPLOY=true
diff --git a/tools/travis/verify_source_code.sh b/tools/travis/verify_source_code.sh
index ef1e1a6..8b14faf 100755
--- a/tools/travis/verify_source_code.sh
+++ b/tools/travis/verify_source_code.sh
@@ -2,8 +2,16 @@
set -e
-cd $TRAVIS_BUILD_DIR
-./tools/travis/download_source_code.sh
+WORK_DIR=${1:-"$(dirname "$TRAVIS_BUILD_DIR")"}
+OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources"
+
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+PARENTDIR="$(dirname "$SCRIPTDIR")"
+
+echo $PARENTDIR
+"$PARENTDIR/install_dependencies.sh"
+"$PARENTDIR/download_source_code.sh" $WORK_DIR
+"$PARENTDIR/verify_source_code.sh" $WORK_DIR
# TODO: verify the source code headers
echo "TODO: verify the source code headers"
diff --git a/tools/upload_artifacts.sh b/tools/upload_artifacts.sh
new file mode 100755
index 0000000..821865a
--- /dev/null
+++ b/tools/upload_artifacts.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "Upload the artifacts."
+
+WORK_DIR=${1:-"$HOME"}
+
+SVN_USERNAME=$2
+SVN_PASSWORD=$3
+CREDENTIALS=""
+
+if [ ! -z "$SVN_USERNAME" ] && [ ! -z "$SVN_PASSWORD" ];then
+ CREDENTIALS="--username $SVN_USERNAME --password $SVN_PASSWORD --non-interactive"
+fi
+
+OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources"
+OPENWHISK_SVN="$OPENWHISK_SOURCE_DIR/openwhisk"
+
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+source "$SCRIPTDIR/util.sh"
+
+CONFIG=$(read_file $SCRIPTDIR/config.json)
+version_key="version"
+version_major=$(json_by_key "$CONFIG" ${version_key}.major)
+version_minor=$(json_by_key "$CONFIG" ${version_key}.minor)
+
+version=$version_major-$version_minor
+CURRENT_VERSION_DIR="$OPENWHISK_SVN/openwhisk-$version"
+
+cd $OPENWHISK_SVN
+svn add --force "openwhisk-$version" $CREDENTIALS
+svn commit -m "Staging Apache OpenWhisk release ${version}." $CREDENTIALS
diff --git a/tools/travis/util.sh b/tools/util.sh
similarity index 100%
rename from tools/travis/util.sh
rename to tools/util.sh
diff --git a/tools/verify_source_code.sh b/tools/verify_source_code.sh
new file mode 100755
index 0000000..7ad4a9d
--- /dev/null
+++ b/tools/verify_source_code.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -e
+
+WORK_DIR=${1:-"$HOME"}
+OPENWHISK_SOURCE_DIR="$WORK_DIR/openwhisk_sources"
+
+SCRIPTDIR="$(cd $(dirname "$0")/ && pwd)"
+
+# TODO: verify the source code headers
+echo "TODO: verify the source code headers"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services