[incubator-heron] branch master updated: Add build scripts for ci jobs (#3174)

[nw...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

nwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git


The following commit(s) were added to refs/heads/master by this push:
     new e82293b  Add build scripts for ci jobs (#3174)
e82293b is described below

commit e82293bc7a1d6514cafbf00eaaae24cb05bbf4ec
Author: Ning Wang <nw...@twitter.com>
AuthorDate: Mon Jan 28 11:56:16 2019 -0800

    Add build scripts for ci jobs (#3174)
---
 scripts/ci/README.md                 | 37 ++++++++++++++
 scripts/ci/build_maven_artifacts.sh  | 93 ++++++++++++++++++++++++++++++++++++
 scripts/ci/build_release_packages.sh | 61 +++++++++++++++++++++++
 scripts/ci/setup_bazel.sh            | 44 +++++++++++++++++
 4 files changed, 235 insertions(+)

diff --git a/scripts/ci/README.md b/scripts/ci/README.md
new file mode 100644
index 0000000..413183f
--- /dev/null
+++ b/scripts/ci/README.md
@@ -0,0 +1,37 @@
+# Heron CI scripts
+This directory contains scripts used by CI jobs.
+
+## Build release packages
+
+Release packages include heron.tar.gz as well as installer script. Packages are platform dependent.
+
+Example:
+
+```bash
+set -e
+set -o pipefail
+
+# Install bazel (linux build) because CI hosts may not have it installed
+sh scripts/ci/setup_bazel.sh linux
+
+# Build v0.20.1-incubating packages for centos7 and put in artifacts folder
+HERON_BUILD_USER=release-agent
+sh scripts/ci/build_release_packages.sh v0.20.1-incubating centos7 artifacts
+
+```
+
+## Build maven artifacts
+
+Maven argifacts include api, spi, storm-compatibility and simulator. Artifacts are platform indepedent.
+
+Example:
+
+```bash
+set -e
+set -o pipefail
+
+# Build v0.20.1-incubating artifacts and put in artifacts folder
+HERON_BUILD_USER=release-agent
+sh scripts/ci/build_maven_artifacts.sh v0.20.1-incubating artifacts
+
+```
diff --git a/scripts/ci/build_maven_artifacts.sh b/scripts/ci/build_maven_artifacts.sh
new file mode 100644
index 0000000..3dc263e
--- /dev/null
+++ b/scripts/ci/build_maven_artifacts.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build all artifacts and put into an artifacts/ folder
+# parameters:
+# 1. version tag, e.g. v0.20.1-incubating
+# 2. output dir
+
+set -e
+set -o pipefail
+
+if [ "$#" -ne 2 ]; then
+    echo "ERROR: Wrong number of arguments. Usage '$0 VERSION_TAG OUTPUT_DIR'"
+    exit 1
+fi
+VERSION_TAG=$1
+OUTPUT_DIR=$2
+
+# Clear out the pex cache
+rm -rf /var/lib/jenkins/.pex/build/*
+
+# Build
+echo "Starting Heron Maven Jars Process"
+bash scripts/release/status.sh
+
+echo "Make jar components"
+$HOME/bin/bazel build heron/api/src/java:all
+$HOME/bin/bazel build heron/spi/src/java:all
+$HOME/bin/bazel build heron/simulator/src/java:all
+$HOME/bin/bazel build storm-compatibility/src/java:all
+
+echo "Found Version Tag $VERSION_TAG"
+mkdir -p artifacts/$VERSION_TAG
+rm -rf artifacts/$VERSION_TAG/*
+
+echo "Run Maven template for poms ... "
+BASE_DIR=`pwd`
+cd ./release/
+
+git clean -f -x .
+sh ./maven/maven-pom-version.sh $VERSION_TAG
+
+cd $BASE_DIR
+
+echo "Build directories for jars ... "
+mkdir $OUTPUT_DIR/$VERSION_TAG/heron-api
+mkdir $OUTPUT_DIR/$VERSION_TAG/heron-spi
+mkdir $OUTPUT_DIR/$VERSION_TAG/heron-simulator
+mkdir $OUTPUT_DIR/$VERSION_TAG/heron-storm
+
+echo "Copy heron-api artifacts ... "
+cp ./release/heron-api-$VERSION_TAG.pom $OUTPUT_DIR/$VERSION_TAG/heron-api/
+cp ./bazel-bin/heron/api/src/java/api-shaded.jar $OUTPUT_DIR/$VERSION_TAG/heron-api/heron-api-$VERSION_TAG.jar
+cp ./bazel-bin/heron/api/src/java/heron-api-javadoc.zip $OUTPUT_DIR/$VERSION_TAG/heron-api/heron-api-$VERSION_TAG-javadoc.jar
+cp ./bazel-bin/heron/api/src/java/libapi-java-low-level-functional-src.jar $OUTPUT_DIR/$VERSION_TAG/heron-api/heron-api-$VERSION_TAG-sources.jar
+
+echo "Copy heron-spi artifacts ... "
+cp ./release/heron-spi-$VERSION_TAG.pom ./$OUTPUT_DIR/$VERSION_TAG/heron-spi/
+cp ./bazel-bin/heron/spi/src/java/spi-unshaded_deploy.jar ./$OUTPUT_DIR/$VERSION_TAG/heron-spi/heron-spi-$VERSION_TAG.jar
+cp ./bazel-bin/heron/spi/src/java/heron-spi-javadoc.zip ./$OUTPUT_DIR/$VERSION_TAG/heron-spi/heron-spi-$VERSION_TAG-javadoc.jar
+cp ./bazel-bin/heron/spi/src/java/libheron-spi-src.jar ./$OUTPUT_DIR/$VERSION_TAG/heron-spi/heron-spi-$VERSION_TAG-sources.jar
+
+echo "Copy heron-simulator artifacts ... "
+cp ./release/heron-simulator-$VERSION_TAG.pom ./$OUTPUT_DIR/$VERSION_TAG/heron-simulator/
+cp ./bazel-bin/heron/simulator/src/java/simulator-shaded.jar ./$OUTPUT_DIR/$VERSION_TAG/heron-simulator/heron-simulator-$VERSION_TAG.jar
+cp ./bazel-bin/heron/simulator/src/java/heron-simulator-javadoc.zip ./$OUTPUT_DIR/$VERSION_TAG/heron-spi/heron-simulator-$VERSION_TAG-javadoc.jar
+cp ./bazel-bin/heron/simulator/src/java/libsimulator-java-src.jar ./$OUTPUT_DIR/$VERSION_TAG/heron-simulator/heron-simulator-$VERSION_TAG-sources.jar
+
+echo "Copy heron-storm artifacts ... "
+cp ./release/heron-storm-$VERSION_TAG.pom ./$OUTPUT_DIR/$VERSION_TAG/heron-storm/
+cp ./bazel-bin/storm-compatibility/src/java/heron-storm.jar ./$OUTPUT_DIR/$VERSION_TAG/heron-storm/heron-storm-$VERSION_TAG.jar
+cp ./bazel-bin/storm-compatibility/src/java/heron-storm-javadoc.zip ./$OUTPUT_DIR/$VERSION_TAG/heron-storm/heron-storm-$VERSION_TAG-javadoc.jar
+cp ./bazel-bin/storm-compatibility/src/java/libstorm-compatibility-java-src.jar ./$OUTPUT_DIR/$VERSION_TAG/heron-storm/heron-storm-$VERSION_TAG-sources.jar
+
+
+echo "Compress all artifacts into a bundle file ..."
+tar -czf "${VERSION_TAG}_artifacts.tar.gz" $OUTPUT_DIR
+mv "${VERSION_TAG}_artifacts.tar.gz" $OUTPUT_DIR
diff --git a/scripts/ci/build_release_packages.sh b/scripts/ci/build_release_packages.sh
new file mode 100644
index 0000000..3e10362
--- /dev/null
+++ b/scripts/ci/build_release_packages.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build packages to be released
+# parameters:
+# 1. version tag, e.g. v0.20.1-incubating
+# 2. build os, e.g. darwin, centos7
+# 3. output dir
+
+# Related environment variables
+# HERON_BUILD_USER
+# HERON_BUILD_HOST
+
+set -e
+set -o pipefail
+
+if [ "$#" -ne 3 ]; then
+    echo "ERROR: Wrong number of arguments. Usage '$0 VERSION_TAG BUILD_OS OUTPUT_DIR'"
+    exit 1
+fi
+VERSION_TAG=$1
+BUILD_OS=$2
+OUTPUT_DIR=$3
+
+HERON_BUILD_VERSION=$VERSION_TAG
+
+# Clear out the pex cache
+rm -rf /var/lib/jenkins/.pex/build/*
+
+
+# Build with docker
+chmod +x scripts/release/status.sh
+chmod +x heron/downloaders/src/shell/heron-downloader.sh
+chmod +x heron/tools/apiserver/src/shell/heron-apiserver.sh
+
+bash scripts/release/status.sh
+
+# Create a temporary director for generated files
+mkdir -p ~/heron-release
+rm -f ~/heron-release/*.*
+./docker/scripts/build-artifacts.sh $BUILD_OS $VERSION_TAG ~/heron-release
+
+# Cherry-pick files to output dir
+mkdir -p $OUTPUT_DIR
+cp ~/heron-release/heron-${VERSION_TAG}-${BUILD_OS}.tar.gz $OUTPUT_DIR
+cp ~/heron-release/heron-install-${VERSION_TAG}-${BUILD_OS}.sh $OUTPUT_DIR
diff --git a/scripts/ci/setup_bazel.sh b/scripts/ci/setup_bazel.sh
new file mode 100644
index 0000000..b38df20
--- /dev/null
+++ b/scripts/ci/setup_bazel.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build all artifacts and put into an artifacts/ folder
+# parameters:
+# 1. version tag, e.g. v0.20.1-incubating
+# 2. output dir
+
+set -e
+set -o pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "ERROR: Wrong number of arguments. Usage '$0 OS[linux|darwin]'"
+    exit 1
+fi
+BAZEL_OS=$1
+BAZEL_VERSION=0.14.1
+
+# Install Bazel
+BAZEL_INSTALLER=bazel-$BAZEL_VERSION-installer-$BAZEL_OS-x86_64.sh
+BAZEL_INSTALLER_LOC="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/$BAZEL_INSTALLER"
+
+wget -O /tmp/$BAZEL_INSTALLER $BAZEL_INSTALLER_LOC
+chmod +x /tmp/$BAZEL_INSTALLER
+/tmp/$BAZEL_INSTALLER --user
+
+chmod +x ./bazel_configure.py
+./bazel_configure.py
+$HOME/bin/bazel clean