[flink] 01/02: [FLINK-9353] Added end to end test for standalone embedded job run in docker

[dw...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit bf0f16ce4cce392d832c1b0c886a4771e7bdd613
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Jul 19 08:51:12 2018 +0200

    [FLINK-9353] Added end to end test for standalone embedded job run in docker
---
 flink-end-to-end-tests/run-pre-commit-tests.sh     |  1 +
 .../container-scripts/docker-compose.test.yml      | 30 +++++++++++++++
 .../test-scripts/test_docker_embedded_job.sh       | 45 ++++++++++++++++++++++
 3 files changed, 76 insertions(+)

diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh
index 6355fd0..2c96392 100755
--- a/flink-end-to-end-tests/run-pre-commit-tests.sh
+++ b/flink-end-to-end-tests/run-pre-commit-tests.sh
@@ -51,6 +51,7 @@ run_test "Shaded Hadoop S3A end-to-end test" "$END_TO_END_DIR/test-scripts/test_
 run_test "Shaded Presto S3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh"
 run_test "Hadoop-free Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_hadoop_free.sh"
 run_test "Distributed cache end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_distributed_cache_via_blob.sh"
+run_test "Wordcount end-to-end test in docker env" "$END_TO_END_DIR/test-scripts/test_docker_embedded_job.sh"
 
 printf "\n[PASS] All tests passed\n"
 exit 0
diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml b/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml
new file mode 100644
index 0000000..5281111
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml
@@ -0,0 +1,30 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Extensions to flink-container/docker/docker-compose.yml that mounts volumes needed for tests
+
+version: "2.1"
+services:
+  job-cluster:
+    volumes:
+      - ${INPUT_VOLUME}:${INPUT_PATH}
+      - ${OUTPUT_VOLUME}:${OUTPUT_PATH}
+  taskmanager:
+    volumes:
+      - ${INPUT_VOLUME}:${INPUT_PATH}
+      - ${OUTPUT_VOLUME}:${OUTPUT_PATH}
diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
new file mode 100755
index 0000000..370ef05
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+source "$(dirname "$0")"/common.sh
+
+DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker
+DOCKER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts
+
+export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount
+export FLINK_DOCKER_IMAGE_NAME=test_docker_embedded_job
+export INPUT_VOLUME=${END_TO_END_DIR}/test-scripts/test-data
+export OUTPUT_VOLUME=${TEST_DATA_DIR}/out
+export INPUT_PATH=/data/test/input
+export OUTPUT_PATH=/data/test/output
+export FLINK_JOB_ARGUMENTS="--input ${INPUT_PATH}/words --output ${OUTPUT_PATH}/docker_wc_out"
+
+# user inside the container must be able to createto workaround in-container permissions
+mkdir -p $OUTPUT_VOLUME
+chmod 777 $OUTPUT_VOLUME
+
+cd "$DOCKER_MODULE_DIR"
+./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME}
+cd "$END_TO_END_DIR"
+
+docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --abort-on-container-exit --exit-code-from job-cluster &> /dev/null
+docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs job-cluster > ${FLINK_DIR}/log/jobmanager.log
+docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs taskmanager > ${FLINK_DIR}/log/taskmanager.log
+
+check_result_hash "WordCount" $OUTPUT_VOLUME/docker_wc_out "72a690412be8928ba239c2da967328a5"