You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2018/11/19 16:37:27 UTC
[flink] branch master updated: [FLINK-10481][e2e] Added retry logic
for building docker image
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 868e411 [FLINK-10481][e2e] Added retry logic for building docker image
868e411 is described below
commit 868e41129639c6d109949f4c1d6d961eed98eb69
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Fri Nov 9 14:58:38 2018 +0100
[FLINK-10481][e2e] Added retry logic for building docker image
---
flink-end-to-end-tests/test-scripts/common.sh | 19 +++++++++++++++++++
.../test-scripts/test_docker_embedded_job.sh | 13 +++++++++++--
2 files changed, 30 insertions(+), 2 deletions(-)
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
index 4e62548..275d9c4 100644
--- a/flink-end-to-end-tests/test-scripts/common.sh
+++ b/flink-end-to-end-tests/test-scripts/common.sh
@@ -663,3 +663,22 @@ function find_latest_completed_checkpoint {
local checkpoint_meta_file=$(ls -d ${checkpoint_root_directory}/chk-[1-9]*/_metadata | sort -Vr | head -n1)
echo "$(dirname "${checkpoint_meta_file}")"
}
+
+function retry_times() {
+ local retriesNumber=$1
+ local backoff=$2
+ local command=${@:3}
+
+ for (( i = 0; i < ${retriesNumber}; i++ ))
+ do
+ if ${command}; then
+ return 0
+ fi
+
+ echo "Command: ${command} failed. Retrying..."
+ sleep ${backoff}
+ done
+
+ echo "Command: ${command} failed ${retriesNumber} times."
+ return 1
+}
diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
index 370ef05..2d8aa4f 100755
--- a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
+++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
@@ -21,6 +21,8 @@ source "$(dirname "$0")"/common.sh
DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker
DOCKER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts
+DOCKER_IMAGE_BUILD_RETRIES=3
+BUILD_BACKOFF_TIME=5
export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount
export FLINK_DOCKER_IMAGE_NAME=test_docker_embedded_job
@@ -30,12 +32,19 @@ export INPUT_PATH=/data/test/input
export OUTPUT_PATH=/data/test/output
export FLINK_JOB_ARGUMENTS="--input ${INPUT_PATH}/words --output ${OUTPUT_PATH}/docker_wc_out"
-# user inside the container must be able to createto workaround in-container permissions
+build_image() {
+ ./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME}
+}
+
+# user inside the container must be able to create files, this is a workaround in-container permissions
mkdir -p $OUTPUT_VOLUME
chmod 777 $OUTPUT_VOLUME
cd "$DOCKER_MODULE_DIR"
-./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME}
+if ! retry_times $DOCKER_IMAGE_BUILD_RETRIES ${BUILD_BACKOFF_TIME} build_image; then
+ echo "Failed to build docker image. Aborting..."
+ exit 1
+fi
cd "$END_TO_END_DIR"
docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --abort-on-container-exit --exit-code-from job-cluster &> /dev/null