You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by li...@apache.org on 2020/07/13 01:31:55 UTC
[submarine] branch master updated: SUBMARINE-553. Modify "set -e"
command in some shell scripts
This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new d8daeab SUBMARINE-553. Modify "set -e" command in some shell scripts
d8daeab is described below
commit d8daeabab02f9b91900595f943649d152161e82f
Author: Ryan Lo <lo...@gmail.com>
AuthorDate: Tue Jul 7 03:24:57 2020 +0800
SUBMARINE-553. Modify "set -e" command in some shell scripts
### What is this PR for?
The command "set -e" is not recommended in shell scripts.
It would be better to be replaced with "set -euo pipefail"
### What type of PR is it?
[Bug Fix]
### Todos
### What is the Jira issue?
[SUBMARINE-553](https://issues.apache.org/jira/projects/SUBMARINE/issues/SUBMARINE-553)
### How should this be tested?
[Travis CI](https://travis-ci.org/github/lowc1012/submarine/builds/704650038)
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Ryan Lo <lo...@gmail.com>
Closes #339 from lowc1012/SUBMARINE-553 and squashes the following commits:
fed1bc9 [Ryan Lo] SUBMARINE-553. Keep set -x in original scripts and update missed files
ab0276e [Ryan Lo] SUBMARINE-553. Modify "set -e" command in some shell scripts
---
dev-support/cicd/build_and_start_cicd_image.sh | 2 +-
dev-support/cicd/common_release.sh | 6 +++---
dev-support/cicd/create_release.sh | 6 +++---
dev-support/cicd/entry.sh | 14 +++++++-------
dev-support/cicd/publish_release.sh | 6 +++---
dev-support/docker-images/database/build.sh | 3 +--
dev-support/docker-images/jupyter/build.sh | 3 +--
dev-support/docker-images/operator/build.sh | 3 +--
dev-support/docker-images/submarine/build.sh | 3 +--
dev-support/k8s/deploy-kubeflow-operators.sh | 10 +++++++---
dev-support/k8s/deploy-notebook-controller.sh | 2 +-
dev-support/mini-submarine/build_mini-submarine.sh | 4 ++--
dev-support/travis/install_external_dependencies.sh | 6 +++---
docs/ecosystem/kaldi/build-all.sh | 2 +-
docs/userdocs/yarn/docker/mxnet/build-all.sh | 2 +-
docs/userdocs/yarn/docker/pytorch/build-all.sh | 2 +-
docs/userdocs/yarn/docker/tensorflow/build-all.sh | 4 ++--
submarine-cloud/build.sh | 2 +-
submarine-cloud/hack/deploy-submarine.sh | 6 ++++--
submarine-cloud/hack/integration-test.sh | 3 +--
submarine-cloud/hack/kind-cluster-build.sh | 2 +-
submarine-sdk/pysubmarine/github-actions/auto-format.sh | 4 ++--
submarine-sdk/pysubmarine/github-actions/lint.sh | 4 ++--
23 files changed, 50 insertions(+), 49 deletions(-)
diff --git a/dev-support/cicd/build_and_start_cicd_image.sh b/dev-support/cicd/build_and_start_cicd_image.sh
index e5b4001..2447c2b 100755
--- a/dev-support/cicd/build_and_start_cicd_image.sh
+++ b/dev-support/cicd/build_and_start_cicd_image.sh
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
+set -euo pipefail
printf "Building Submarine CI/CD Image.\n"
docker build -t submarine-cicd .
diff --git a/dev-support/cicd/common_release.sh b/dev-support/cicd/common_release.sh
index c597121..995d6f4 100755
--- a/dev-support/cicd/common_release.sh
+++ b/dev-support/cicd/common_release.sh
@@ -19,15 +19,15 @@
# common functions
-if [[ -z "${TAR}" ]]; then
+if [[ -z "${TAR:-}" ]]; then
TAR="/usr/bin/tar"
fi
-if [[ -z "${SHASUM}" ]]; then
+if [[ -z "${SHASUM:-}" ]]; then
SHASUM="/usr/bin/shasum"
fi
-if [[ -z "${WORKING_DIR}" ]]; then
+if [[ -z "${WORKING_DIR:-}" ]]; then
WORKING_DIR="/tmp/submarine-release"
fi
diff --git a/dev-support/cicd/create_release.sh b/dev-support/cicd/create_release.sh
index 418f800..f4621d3 100755
--- a/dev-support/cicd/create_release.sh
+++ b/dev-support/cicd/create_release.sh
@@ -25,7 +25,7 @@
# http://www.apache.org/dev/release-publishing
# http://www.apache.org/dev/release-signing.html
-set -e
+set -euo pipefail
BASEDIR="$(dirname "$0")"
. "${BASEDIR}/common_release.sh"
@@ -36,7 +36,7 @@ if [[ $# -ne 2 ]]; then
fi
for var in GPG_PASSPHRASE; do
- if [[ -z "${!var}" ]]; then
+ if [[ -z "${!var:-}" ]]; then
echo "You need ${var} variable set"
exit 1
fi
@@ -52,7 +52,7 @@ function compile_src_and_bin() {
echo "mvn clean install package -DskipTests -Psrc"
mvn clean install package -DskipTests -Psrc
if [[ $? -ne 0 ]]; then
- echo "Build failed. ${BUILD_FLAGS}"
+ echo "Build failed. ${BUILD_FLAGS:-}"
exit 1
fi
diff --git a/dev-support/cicd/entry.sh b/dev-support/cicd/entry.sh
index e75a12b..a6a1394 100755
--- a/dev-support/cicd/entry.sh
+++ b/dev-support/cicd/entry.sh
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
+set -euo pipefail
# activate python 2.7.13 environment
. ${PYTHON_VENV_PATH}/venv2.7/bin/activate
@@ -42,20 +42,20 @@ function merge_pr(){
apache_id="id"
apache_name="name"
- if [ -z "$JIRA_USERNAME" ]; then
+ if [ -z "${JIRA_USERNAME:-}" ]; then
read -p "Enter Your Apache JIRA User name: " jira_name
else
jira_name=$JIRA_USERNAME
fi
echo "Got JIRA name: ${jira_name}"
- if [ -z "$JIRA_PASSWORD" ]; then
+ if [ -z "${JIRA_PASSWORD:-}" ]; then
read -s -p "Enter Your Apache JIRA User passwd: " jira_pwd
else
jira_pwd=$JIRA_PASSWORD
fi
- if [ -z "$APACHE_ID" ]; then
+ if [ -z "${APACHE_ID:-}" ]; then
printf "\n"
read -p "Enter Your Apache committer ID: " apache_id
else
@@ -63,7 +63,7 @@ function merge_pr(){
fi
echo "Got Apache ID: ${apache_id}"
- if [ -z "$APACHE_NAME" ]; then
+ if [ -z "${APACHE_NAME:-}" ]; then
read -p "Enter Your Apache committer name: " apache_name
else
apache_name=$APACHE_NAME
@@ -86,7 +86,7 @@ function update_submarine_site(){
apache_id="id"
apache_name="name"
- if [ -z "$APACHE_ID" ]; then
+ if [ -z "${APACHE_ID:-}" ]; then
printf "\n"
read -p "Enter Your Apache committer ID: " apache_id
else
@@ -94,7 +94,7 @@ function update_submarine_site(){
fi
echo "Got Apache ID: ${apache_id}"
- if [ -z "$APACHE_NAME" ]; then
+ if [ -z "${APACHE_NAME:-}" ]; then
read -p "Enter Your Apache committer name: " apache_name
else
apache_name=$APACHE_NAME
diff --git a/dev-support/cicd/publish_release.sh b/dev-support/cicd/publish_release.sh
index 275e052..1a1ee90 100755
--- a/dev-support/cicd/publish_release.sh
+++ b/dev-support/cicd/publish_release.sh
@@ -21,7 +21,7 @@
#
# Here's some helpful documents for the release.
# http://www.apache.org/dev/publishing-maven-artifacts.html
-set -e
+set -euo pipefail
BASEDIR="$(dirname "$0")"
. "${BASEDIR}/common_release.sh"
@@ -30,7 +30,7 @@ if [[ $# -ne 2 ]]; then
fi
for var in GPG_PASSPHRASE ASF_USERID ASF_PASSWORD; do
- if [[ -z "${!var}" ]]; then
+ if [[ -z "${!var:-}" ]]; then
echo "You need ${var} variable set"
exit 1
fi
@@ -164,7 +164,7 @@ function publish_to_maven() {
}
git_clone
-if [[ "${DO_SNAPSHOT}" == 'yes' ]]; then
+if [[ "${DO_SNAPSHOT:-}" == 'yes' ]]; then
publish_snapshot_to_maven
else
publish_to_maven
diff --git a/dev-support/docker-images/database/build.sh b/dev-support/docker-images/database/build.sh
index e59600a..7b463ba 100755
--- a/dev-support/docker-images/database/build.sh
+++ b/dev-support/docker-images/database/build.sh
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -eo pipefail
-set -x
+set -euxo pipefail
if [ -L ${BASH_SOURCE-$0} ]; then
PWD=$(dirname $(readlink "${BASH_SOURCE-$0}"))
diff --git a/dev-support/docker-images/jupyter/build.sh b/dev-support/docker-images/jupyter/build.sh
index 19dc61d..200f0eb 100755
--- a/dev-support/docker-images/jupyter/build.sh
+++ b/dev-support/docker-images/jupyter/build.sh
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -eo pipefail
-set -x
+set -euxo pipefail
TF_JUPYTER_IMAGE="apache/submarine:tf2.1.0-jupyter"
diff --git a/dev-support/docker-images/operator/build.sh b/dev-support/docker-images/operator/build.sh
index 72c212d..f015980 100755
--- a/dev-support/docker-images/operator/build.sh
+++ b/dev-support/docker-images/operator/build.sh
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -eo pipefail
-set -x
+set -euxo pipefail
SUBMARINE_VERSION=0.5.0-SNAPSHOT
SUBMARINE_IMAGE_NAME="apache/submarine:operator-${SUBMARINE_VERSION}"
diff --git a/dev-support/docker-images/submarine/build.sh b/dev-support/docker-images/submarine/build.sh
index 1716a38..87f2ed5 100755
--- a/dev-support/docker-images/submarine/build.sh
+++ b/dev-support/docker-images/submarine/build.sh
@@ -14,8 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -eo pipefail
-set -x
+set -euxo pipefail
SUBMARINE_VERSION=0.5.0-SNAPSHOT
SUBMARINE_IMAGE_NAME="apache/submarine:server-${SUBMARINE_VERSION}"
diff --git a/dev-support/k8s/deploy-kubeflow-operators.sh b/dev-support/k8s/deploy-kubeflow-operators.sh
index d35ef86..26ad872 100755
--- a/dev-support/k8s/deploy-kubeflow-operators.sh
+++ b/dev-support/k8s/deploy-kubeflow-operators.sh
@@ -16,7 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -e
+set -euo pipefail
readonly TF_OPERATOR_IMAGE="gcr.io/kubeflow-images-public/tf_operator:v1.0.0-g92389064"
readonly PYTORCH_OPERATOR_IMAGE="gcr.io/kubeflow-images-public/pytorch-operator:v1.0.0-g047cf0f"
@@ -68,7 +68,7 @@ function deploy_tf_operator() {
${KUBECTL_BIN} kustomize "${CURRENT_PATH}/tfjob/operator" \
| ${KUBECTL_BIN} apply -f -
- if [[ $1 == "true" ]]; then
+ if [[ ${1:-} == "true" ]]; then
load_image_to_registry "${TF_MNIST_IMAGE}"
fi
}
@@ -86,7 +86,7 @@ function deploy_pytorch_operator() {
load_image_to_registry "${PYTORCH_OPERATOR_IMAGE}"
${KUBECTL_BIN} apply -f "${CURRENT_PATH}/pytorchjob"
- if [[ $1 == "true" ]]; then
+ if [[ ${1:-} == "true" ]]; then
load_image_to_registry "${PT_MNIST_IMAGE}"
fi
}
@@ -143,6 +143,10 @@ function main() {
esac
done
+ opt_all=${opt_all:-}
+ opt_tf=${opt_tf:-}
+ opt_pt=${opt_pt:-}
+ opt_s=${opt_s:-}
hack::ensure_kubectl
if [[ "${opt_tf}" == "true" && "${opt_pt}" == "true" ]]; then
diff --git a/dev-support/k8s/deploy-notebook-controller.sh b/dev-support/k8s/deploy-notebook-controller.sh
index a672daa..d983ca5 100755
--- a/dev-support/k8s/deploy-notebook-controller.sh
+++ b/dev-support/k8s/deploy-notebook-controller.sh
@@ -16,7 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -e
+set -euo pipefail
readonly NOTEBOOK_CONTROLLER_IMAGE="gcr.io/kubeflow-images-public/notebook-controller:v1.1.0-g253890cb"
diff --git a/dev-support/mini-submarine/build_mini-submarine.sh b/dev-support/mini-submarine/build_mini-submarine.sh
index a8641a9..0c13b4a 100755
--- a/dev-support/mini-submarine/build_mini-submarine.sh
+++ b/dev-support/mini-submarine/build_mini-submarine.sh
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
+set -euo pipefail
hadoop_v=2.9.2
spark_v=2.3.4
@@ -66,7 +66,7 @@ fi
submarine_dist_exists=$(find -L "${SUBMARINE_PROJECT_PATH}/submarine-dist/target" -name "submarine-dist-${submarine_v}*.tar.gz")
# If exists, use the release candidate artifacts to build image
-if [[ ! -z "${release_candidates_path}" ]]; then
+if [[ ! -z "${release_candidates_path:-}" ]]; then
submarine_dist_exists=${release_candidates_path}
echo "Using release candidates artifacts: ${release_candidates_path}"
cp ${release_candidates_path}/submarine-dist-${submarine_v}-hadoop*.tar.gz ${MINI_PATH}
diff --git a/dev-support/travis/install_external_dependencies.sh b/dev-support/travis/install_external_dependencies.sh
index aa749c5..8a77fe2 100755
--- a/dev-support/travis/install_external_dependencies.sh
+++ b/dev-support/travis/install_external_dependencies.sh
@@ -17,11 +17,11 @@
#
# Script for installing R / Python dependencies for Travis CI
-set -ev
+set -evuo pipefail
touch ~/.environ
# Install Python dependencies for Python specific tests
-if [[ -n "$PYTHON" ]] ; then
+if [[ -n "${PYTHON:-}" ]] ; then
wget https://repo.continuum.io/miniconda/Miniconda${PYTHON}-4.2.12-Linux-x86_64.sh -O miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
echo "export PATH='$HOME/miniconda/bin:$PATH'" >> ~/.environ
@@ -36,7 +36,7 @@ if [[ -n "$PYTHON" ]] ; then
conda install -q numpy=1.13.3 pandas=0.21.1 matplotlib=2.1.1 pandasql=0.7.3 ipython=5.4.1 jupyter_client=5.1.0 ipykernel=4.7.0 bokeh=0.12.10
pip install -q scipy==0.18.0 ggplot==0.11.5 grpcio==1.8.2 bkzep==0.4.0
- if [[ -n "$TENSORFLOW" ]] ; then
+ if [[ -n "${TENSORFLOW:-}" ]] ; then
check_results=`conda search -c conda-forge tensorflow`
echo "search tensorflow = $check_results"
diff --git a/docs/ecosystem/kaldi/build-all.sh b/docs/ecosystem/kaldi/build-all.sh
index 020876e..61c590b 100755
--- a/docs/ecosystem/kaldi/build-all.sh
+++ b/docs/ecosystem/kaldi/build-all.sh
@@ -17,7 +17,7 @@
echo "Building base images"
-set -e
+set -euo pipefail
cd base/ubuntu-18.04
diff --git a/docs/userdocs/yarn/docker/mxnet/build-all.sh b/docs/userdocs/yarn/docker/mxnet/build-all.sh
index 45f5767..5aab922 100755
--- a/docs/userdocs/yarn/docker/mxnet/build-all.sh
+++ b/docs/userdocs/yarn/docker/mxnet/build-all.sh
@@ -16,7 +16,7 @@
echo "Building base images"
-set -e
+set -euo pipefail
cd base/ubuntu-18.04
diff --git a/docs/userdocs/yarn/docker/pytorch/build-all.sh b/docs/userdocs/yarn/docker/pytorch/build-all.sh
index 872d32b..f0f795c 100755
--- a/docs/userdocs/yarn/docker/pytorch/build-all.sh
+++ b/docs/userdocs/yarn/docker/pytorch/build-all.sh
@@ -17,7 +17,7 @@
echo "Building base images"
-set -e
+set -euo pipefail
cd base/ubuntu-18.04
diff --git a/docs/userdocs/yarn/docker/tensorflow/build-all.sh b/docs/userdocs/yarn/docker/tensorflow/build-all.sh
index db1704f..5c02379 100755
--- a/docs/userdocs/yarn/docker/tensorflow/build-all.sh
+++ b/docs/userdocs/yarn/docker/tensorflow/build-all.sh
@@ -17,7 +17,7 @@
echo "Building base images"
-set -e
+set -euo pipefail
cd base/ubuntu-18.04
@@ -32,4 +32,4 @@ docker build . -f Dockerfile.cpu.tf_1.13.1 -t tf-1.13.1-cpu:0.0.1
docker build . -f Dockerfile.gpu.tf_1.13.1 -t tf-1.13.1-gpu:0.0.1
cd ../../mnist
-docker build . -f Dockerfile.tony.tf.mnist.tf_1.13.1 -t tony-mnist-tf-1.13.1:0.0.1
\ No newline at end of file
+docker build . -f Dockerfile.tony.tf.mnist.tf_1.13.1 -t tony-mnist-tf-1.13.1:0.0.1
diff --git a/submarine-cloud/build.sh b/submarine-cloud/build.sh
index 03f910b..9890a1e 100755
--- a/submarine-cloud/build.sh
+++ b/submarine-cloud/build.sh
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -e
+set -euo pipefail
if [ -L ${BASH_SOURCE-$0} ]; then
PWD=$(dirname $(readlink "${BASH_SOURCE-$0}"))
diff --git a/submarine-cloud/hack/deploy-submarine.sh b/submarine-cloud/hack/deploy-submarine.sh
index 2576a16..7b340da 100755
--- a/submarine-cloud/hack/deploy-submarine.sh
+++ b/submarine-cloud/hack/deploy-submarine.sh
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -e
+set -euo pipefail
ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
cd $ROOT
@@ -136,12 +136,14 @@ case $key in
esac
done
+TEST=${TEST:-}
+UNINSTALL=${UNINSTALL:-}
DATABASE_IP=${DATABASE_IP:-submarine-database}
echo "Submarine database ip: ${DATABASE_IP}"
export KUBECONFIG=~/.kube/kind-config-${clusterName:-kind}
-if [[ "$UNINSTALL" == "TRUE" ]]; then
+if [[ "${UNINSTALL}" == "TRUE" ]]; then
uninstall_submarine
else
install_submarine
diff --git a/submarine-cloud/hack/integration-test.sh b/submarine-cloud/hack/integration-test.sh
index 51a6723..0ca5595 100755
--- a/submarine-cloud/hack/integration-test.sh
+++ b/submarine-cloud/hack/integration-test.sh
@@ -15,8 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -eo pipefail
-set -x
+set -euxo pipefail
ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
cd $ROOT
diff --git a/submarine-cloud/hack/kind-cluster-build.sh b/submarine-cloud/hack/kind-cluster-build.sh
index b252f25..4129387 100755
--- a/submarine-cloud/hack/kind-cluster-build.sh
+++ b/submarine-cloud/hack/kind-cluster-build.sh
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-set -e
+set -euo pipefail
ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
cd $ROOT
diff --git a/submarine-sdk/pysubmarine/github-actions/auto-format.sh b/submarine-sdk/pysubmarine/github-actions/auto-format.sh
index 1609e92..b3327bb 100755
--- a/submarine-sdk/pysubmarine/github-actions/auto-format.sh
+++ b/submarine-sdk/pysubmarine/github-actions/auto-format.sh
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -ex
+set -euxo pipefail
FWDIR="$(cd "$(dirname "$0")"; pwd)"
cd "$FWDIR"
@@ -25,4 +25,4 @@ yapf -i submarine/**/*.py tests/**/*.py
# Sort imports
isort submarine/**/*.py tests/**/*.py
-set +ex
+set +euxo pipefail
diff --git a/submarine-sdk/pysubmarine/github-actions/lint.sh b/submarine-sdk/pysubmarine/github-actions/lint.sh
index 2bbcb13..9e17c8e 100755
--- a/submarine-sdk/pysubmarine/github-actions/lint.sh
+++ b/submarine-sdk/pysubmarine/github-actions/lint.sh
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -ex
+set -euxo pipefail
FWDIR="$(cd "$(dirname "$0")"; pwd)"
cd "$FWDIR"
@@ -40,4 +40,4 @@ else
echo "Test successful"
fi
-set +ex
+set +euxo pipefail
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org