You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@heron.apache.org by nw...@apache.org on 2019/01/31 01:32:58 UTC
[incubator-heron] branch master updated: Add ci script for building
docker image (#3181)
This is an automated email from the ASF dual-hosted git repository.
nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 9b846ec Add ci script for building docker image (#3181)
9b846ec is described below
commit 9b846ec8b6b7e58d3f2c8a0d90cbaa57b7454ff2
Author: Ning Wang <nw...@twitter.com>
AuthorDate: Wed Jan 30 17:32:53 2019 -0800
Add ci script for building docker image (#3181)
* Add ci script for building docker image
---
scripts/ci/README.md | 28 +++++++++++++++++++---
...d_release_packages.sh => build_docker_image.sh} | 23 +++++++++++-------
scripts/ci/build_maven_artifacts.sh | 12 +++++-----
scripts/ci/build_release_packages.sh | 17 +++++++------
4 files changed, 55 insertions(+), 25 deletions(-)
diff --git a/scripts/ci/README.md b/scripts/ci/README.md
index 413183f..7876f5a 100644
--- a/scripts/ci/README.md
+++ b/scripts/ci/README.md
@@ -12,11 +12,11 @@ set -e
set -o pipefail
# Install bazel (linux build) because CI hosts may not have it installed
-sh scripts/ci/setup_bazel.sh linux
+bash scripts/ci/setup_bazel.sh linux
# Build v0.20.1-incubating packages for centos7 and put in artifacts folder
HERON_BUILD_USER=release-agent
-sh scripts/ci/build_release_packages.sh v0.20.1-incubating centos7 artifacts
+bash scripts/ci/build_release_packages.sh v0.20.1-incubating centos7 artifacts
```
@@ -30,8 +30,30 @@ Example:
set -e
set -o pipefail
+# Install bazel (linux build) because CI hosts may not have it installed
+bash scripts/ci/setup_bazel.sh linux
+
+# Build v0.20.1-incubating artifacts and put in artifacts folder
+HERON_BUILD_USER=release-agent
+bash scripts/ci/build_maven_artifacts.sh v0.20.1-incubating artifacts
+
+```
+
+## Build docker image
+
+The docker image includes Heron core, tools and examples.
+
+Example:
+
+```bash
+set -e
+set -o pipefail
+
+# Install bazel (linux build) because CI hosts may not have it installed
+bash scripts/ci/setup_bazel.sh linux
+
# Build v0.20.1-incubating artifacts and put in artifacts folder
HERON_BUILD_USER=release-agent
-sh scripts/ci/build_maven_artifacts.sh v0.20.1-incubating artifacts
+bash scripts/ci/build_docker_image.sh v0.20.1-incubating debian9 artifacts
```
diff --git a/scripts/ci/build_release_packages.sh b/scripts/ci/build_docker_image.sh
similarity index 73%
copy from scripts/ci/build_release_packages.sh
copy to scripts/ci/build_docker_image.sh
index 3e10362..c43fa90 100644
--- a/scripts/ci/build_release_packages.sh
+++ b/scripts/ci/build_docker_image.sh
@@ -16,10 +16,10 @@
# specific language governing permissions and limitations
# under the License.
-# Build packages to be released
+# Build docker image to be released
# parameters:
# 1. version tag, e.g. v0.20.1-incubating
-# 2. build os, e.g. darwin, centos7
+# 2. build os, e.g. debian9, centos7
# 3. output dir
# Related environment variables
@@ -37,7 +37,7 @@ VERSION_TAG=$1
BUILD_OS=$2
OUTPUT_DIR=$3
-HERON_BUILD_VERSION=$VERSION_TAG
+TEMP_RELEASE_DIR=~/heron-release
# Clear out the pex cache
rm -rf /var/lib/jenkins/.pex/build/*
@@ -50,12 +50,17 @@ chmod +x heron/tools/apiserver/src/shell/heron-apiserver.sh
bash scripts/release/status.sh
-# Create a temporary director for generated files
-mkdir -p ~/heron-release
-rm -f ~/heron-release/*.*
-./docker/scripts/build-artifacts.sh $BUILD_OS $VERSION_TAG ~/heron-release
+# Create a temporary directory for generated files
+mkdir -p $TEMP_RELEASE_DIR
+rm -f $TEMP_RELEASE_DIR/*.*
+
+# Build artifacts and the docker image
+./docker/scripts/build-artifacts.sh $BUILD_OS $VERSION_TAG $TEMP_RELEASE_DIR
+./docker/scripts/build-docker.sh $BUILD_OS $VERSION_TAG $TEMP_RELEASE_DIR
# Cherry-pick files to output dir
mkdir -p $OUTPUT_DIR
-cp ~/heron-release/heron-${VERSION_TAG}-${BUILD_OS}.tar.gz $OUTPUT_DIR
-cp ~/heron-release/heron-install-${VERSION_TAG}-${BUILD_OS}.sh $OUTPUT_DIR
+cp ${TEMP_RELEASE_DIR}/heron-docker-${VERSION_TAG}-${BUILD_OS}.tar.gz $OUTPUT_DIR
+
+# Remove temporary directory
+rm -rf $TEMP_RELEASE_DIR
diff --git a/scripts/ci/build_maven_artifacts.sh b/scripts/ci/build_maven_artifacts.sh
index 3dc263e..fdf469c 100644
--- a/scripts/ci/build_maven_artifacts.sh
+++ b/scripts/ci/build_maven_artifacts.sh
@@ -58,10 +58,10 @@ sh ./maven/maven-pom-version.sh $VERSION_TAG
cd $BASE_DIR
echo "Build directories for jars ... "
-mkdir $OUTPUT_DIR/$VERSION_TAG/heron-api
-mkdir $OUTPUT_DIR/$VERSION_TAG/heron-spi
-mkdir $OUTPUT_DIR/$VERSION_TAG/heron-simulator
-mkdir $OUTPUT_DIR/$VERSION_TAG/heron-storm
+mkdir -p $OUTPUT_DIR/$VERSION_TAG/heron-api
+mkdir -p $OUTPUT_DIR/$VERSION_TAG/heron-spi
+mkdir -p $OUTPUT_DIR/$VERSION_TAG/heron-simulator
+mkdir -p $OUTPUT_DIR/$VERSION_TAG/heron-storm
echo "Copy heron-api artifacts ... "
cp ./release/heron-api-$VERSION_TAG.pom $OUTPUT_DIR/$VERSION_TAG/heron-api/
@@ -89,5 +89,5 @@ cp ./bazel-bin/storm-compatibility/src/java/libstorm-compatibility-java-src.jar
echo "Compress all artifacts into a bundle file ..."
-tar -czf "${VERSION_TAG}_artifacts.tar.gz" $OUTPUT_DIR
-mv "${VERSION_TAG}_artifacts.tar.gz" $OUTPUT_DIR
+tar -czf "heron-artifacts-${VERSION_TAG}.tar.gz" $OUTPUT_DIR
+mv "heron-artifacts-${VERSION_TAG}.tar.gz" $OUTPUT_DIR
diff --git a/scripts/ci/build_release_packages.sh b/scripts/ci/build_release_packages.sh
index 3e10362..1e38cf0 100644
--- a/scripts/ci/build_release_packages.sh
+++ b/scripts/ci/build_release_packages.sh
@@ -37,7 +37,7 @@ VERSION_TAG=$1
BUILD_OS=$2
OUTPUT_DIR=$3
-HERON_BUILD_VERSION=$VERSION_TAG
+TEMP_RELEASE_DIR=~/heron-release
# Clear out the pex cache
rm -rf /var/lib/jenkins/.pex/build/*
@@ -50,12 +50,15 @@ chmod +x heron/tools/apiserver/src/shell/heron-apiserver.sh
bash scripts/release/status.sh
-# Create a temporary director for generated files
-mkdir -p ~/heron-release
-rm -f ~/heron-release/*.*
-./docker/scripts/build-artifacts.sh $BUILD_OS $VERSION_TAG ~/heron-release
+# Create a temporary directory for generated files
+mkdir -p $TEMP_RELEASE_DIR
+rm -f $TEMP_RELEASE_DIR/*.*
+./docker/scripts/build-artifacts.sh $BUILD_OS $VERSION_TAG $TEMP_RELEASE_DIR
# Cherry-pick files to output dir
mkdir -p $OUTPUT_DIR
-cp ~/heron-release/heron-${VERSION_TAG}-${BUILD_OS}.tar.gz $OUTPUT_DIR
-cp ~/heron-release/heron-install-${VERSION_TAG}-${BUILD_OS}.sh $OUTPUT_DIR
+cp $TEMP_RELEASE_DIR/heron-${VERSION_TAG}-${BUILD_OS}.tar.gz $OUTPUT_DIR
+cp $TEMP_RELEASE_DIR/heron-install-${VERSION_TAG}-${BUILD_OS}.sh $OUTPUT_DIR
+
+# Remove temporary directory
+rm -rf $TEMP_RELEASE_DIR