You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2018/07/26 09:46:55 UTC
[flink] branch master updated (724857c -> 1a0137e)
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 724857c [FLINK-9916] Add FROM_BASE64 function for table/sql API
new bf0f16c [FLINK-9353] Added end to end test for standalone embedded job run in docker
new 1a0137e [FLINK-9353] Added end to end test for standalone embedded job in kubernetes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
flink-end-to-end-tests/README.md | 4 ++
flink-end-to-end-tests/run-pre-commit-tests.sh | 1 +
.../container-scripts/docker-compose.test.yml | 18 ++++---
.../job-cluster-job.yaml.template | 4 +-
.../task-manager-deployment.yaml.template | 7 +++
.../test-scripts/test_docker_embedded_job.sh | 45 +++++++++++++++++
.../test-scripts/test_kubernetes_embedded_job.sh | 56 ++++++++++++++++++++++
7 files changed, 127 insertions(+), 8 deletions(-)
copy flink-clients/src/test/resources/invalidtestconfig/flink-conf.yaml => flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml (74%)
copy {flink-container/kubernetes => flink-end-to-end-tests/test-scripts/container-scripts}/job-cluster-job.yaml.template (92%)
copy {flink-container/kubernetes => flink-end-to-end-tests/test-scripts/container-scripts}/task-manager-deployment.yaml.template (88%)
create mode 100755 flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
create mode 100755 flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh
[flink] 01/02: [FLINK-9353] Added end to end test for standalone
embedded job run in docker
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit bf0f16ce4cce392d832c1b0c886a4771e7bdd613
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Jul 19 08:51:12 2018 +0200
[FLINK-9353] Added end to end test for standalone embedded job run in docker
---
flink-end-to-end-tests/run-pre-commit-tests.sh | 1 +
.../container-scripts/docker-compose.test.yml | 30 +++++++++++++++
.../test-scripts/test_docker_embedded_job.sh | 45 ++++++++++++++++++++++
3 files changed, 76 insertions(+)
diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh
index 6355fd0..2c96392 100755
--- a/flink-end-to-end-tests/run-pre-commit-tests.sh
+++ b/flink-end-to-end-tests/run-pre-commit-tests.sh
@@ -51,6 +51,7 @@ run_test "Shaded Hadoop S3A end-to-end test" "$END_TO_END_DIR/test-scripts/test_
run_test "Shaded Presto S3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh"
run_test "Hadoop-free Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_hadoop_free.sh"
run_test "Distributed cache end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_distributed_cache_via_blob.sh"
+run_test "Wordcount end-to-end test in docker env" "$END_TO_END_DIR/test-scripts/test_docker_embedded_job.sh"
printf "\n[PASS] All tests passed\n"
exit 0
diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml b/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml
new file mode 100644
index 0000000..5281111
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/container-scripts/docker-compose.test.yml
@@ -0,0 +1,30 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Extensions to flink-container/docker/docker-compose.yml that mounts volumes needed for tests
+
+version: "2.1"
+services:
+ job-cluster:
+ volumes:
+ - ${INPUT_VOLUME}:${INPUT_PATH}
+ - ${OUTPUT_VOLUME}:${OUTPUT_PATH}
+ taskmanager:
+ volumes:
+ - ${INPUT_VOLUME}:${INPUT_PATH}
+ - ${OUTPUT_VOLUME}:${OUTPUT_PATH}
diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
new file mode 100755
index 0000000..370ef05
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+source "$(dirname "$0")"/common.sh
+
+DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker
+DOCKER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts
+
+export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount
+export FLINK_DOCKER_IMAGE_NAME=test_docker_embedded_job
+export INPUT_VOLUME=${END_TO_END_DIR}/test-scripts/test-data
+export OUTPUT_VOLUME=${TEST_DATA_DIR}/out
+export INPUT_PATH=/data/test/input
+export OUTPUT_PATH=/data/test/output
+export FLINK_JOB_ARGUMENTS="--input ${INPUT_PATH}/words --output ${OUTPUT_PATH}/docker_wc_out"
+
+# user inside the container must be able to createto workaround in-container permissions
+mkdir -p $OUTPUT_VOLUME
+chmod 777 $OUTPUT_VOLUME
+
+cd "$DOCKER_MODULE_DIR"
+./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_DOCKER_IMAGE_NAME}
+cd "$END_TO_END_DIR"
+
+docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml up --abort-on-container-exit --exit-code-from job-cluster &> /dev/null
+docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs job-cluster > ${FLINK_DIR}/log/jobmanager.log
+docker-compose -f ${DOCKER_MODULE_DIR}/docker-compose.yml -f ${DOCKER_SCRIPTS}/docker-compose.test.yml logs taskmanager > ${FLINK_DIR}/log/taskmanager.log
+
+check_result_hash "WordCount" $OUTPUT_VOLUME/docker_wc_out "72a690412be8928ba239c2da967328a5"
[flink] 02/02: [FLINK-9353] Added end to end test for standalone
embedded job in kubernetes
Posted by dw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 1a0137e246f40a357bf187d47038b596b1842afa
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Jul 19 08:51:55 2018 +0200
[FLINK-9353] Added end to end test for standalone embedded job in kubernetes
---
flink-end-to-end-tests/README.md | 4 ++
.../job-cluster-job.yaml.template | 46 ++++++++++++++++++
.../task-manager-deployment.yaml.template | 41 ++++++++++++++++
.../test-scripts/test_kubernetes_embedded_job.sh | 56 ++++++++++++++++++++++
4 files changed, 147 insertions(+)
diff --git a/flink-end-to-end-tests/README.md b/flink-end-to-end-tests/README.md
index 26dba3d..257298e 100644
--- a/flink-end-to-end-tests/README.md
+++ b/flink-end-to-end-tests/README.md
@@ -31,6 +31,10 @@ You can also run tests individually via
$ FLINK_DIR=<flink dir> flink-end-to-end-tests/run-single-test.sh your_test.sh arg1 arg2
```
+### Kubernetes test
+
+Kubernetes test (test_kubernetes_embedded_job.sh) assumes a running minikube cluster.
+
## Writing Tests
### Examples
diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template b/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template
new file mode 100644
index 0000000..3785e6f
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/container-scripts/job-cluster-job.yaml.template
@@ -0,0 +1,46 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: flink-job-cluster
+spec:
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: job-cluster
+ spec:
+ restartPolicy: OnFailure
+ containers:
+ - name: flink-job-cluster
+ image: ${FLINK_IMAGE_NAME}
+ imagePullPolicy: Never
+ args: ["job-cluster", "--job-classname", "${FLINK_JOB}", "-Djobmanager.rpc.address=flink-job-cluster",
+ "-Dparallelism.default=${FLINK_JOB_PARALLELISM}", "-Dblob.server.port=6124", "-Dquery.server.ports=6125",
+ ${FLINK_JOB_ARGUMENTS}]
+ ports:
+ - containerPort: 6123
+ name: rpc
+ - containerPort: 6124
+ name: blob
+ - containerPort: 6125
+ name: query
+ - containerPort: 8081
+ name: ui
diff --git a/flink-end-to-end-tests/test-scripts/container-scripts/task-manager-deployment.yaml.template b/flink-end-to-end-tests/test-scripts/container-scripts/task-manager-deployment.yaml.template
new file mode 100644
index 0000000..40d1d27
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/container-scripts/task-manager-deployment.yaml.template
@@ -0,0 +1,41 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: flink-task-manager
+spec:
+ replicas: ${FLINK_JOB_PARALLELISM}
+ template:
+ metadata:
+ labels:
+ app: flink
+ component: task-manager
+ spec:
+ containers:
+ - name: flink-task-manager
+ image: ${FLINK_IMAGE_NAME}
+ imagePullPolicy: Never
+ args: ["task-manager", "-Djobmanager.rpc.address=flink-job-cluster"]
+ volumeMounts:
+ - mountPath: /cache
+ name: cache-volume
+ volumes:
+ - name: cache-volume
+ emptyDir: {}
diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh
new file mode 100755
index 0000000..4529fd9
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+source "$(dirname "$0")"/common.sh
+
+DOCKER_MODULE_DIR=${END_TO_END_DIR}/../flink-container/docker
+KUBERNETES_MODULE_DIR=${END_TO_END_DIR}/../flink-container/kubernetes
+CONTAINER_SCRIPTS=${END_TO_END_DIR}/test-scripts/container-scripts
+
+export FLINK_JOB=org.apache.flink.examples.java.wordcount.WordCount
+export FLINK_IMAGE_NAME=test_kubernetes_embedded_job
+export OUTPUT_VOLUME=${TEST_DATA_DIR}/out
+export OUTPUT_FILE=kubernetes_wc_out
+export FLINK_JOB_PARALLELISM=1
+export FLINK_JOB_ARGUMENTS='"--output", "/cache/kubernetes_wc_out"'
+
+function cleanup {
+ kubectl delete job flink-job-cluster
+ kubectl delete service flink-job-cluster
+ kubectl delete deployment flink-task-manager
+ rm -rf ${OUTPUT_VOLUME}
+}
+
+trap cleanup EXIT
+
+mkdir -p $OUTPUT_VOLUME
+
+eval $(minikube docker-env)
+cd "$DOCKER_MODULE_DIR"
+./build.sh --from-local-dist --job-jar ${FLINK_DIR}/examples/batch/WordCount.jar --image-name ${FLINK_IMAGE_NAME}
+cd "$END_TO_END_DIR"
+
+
+kubectl create -f ${KUBERNETES_MODULE_DIR}/job-cluster-service.yaml
+envsubst '${FLINK_IMAGE_NAME} ${FLINK_JOB} ${FLINK_JOB_PARALLELISM} ${FLINK_JOB_ARGUMENTS}' < ${CONTAINER_SCRIPTS}/job-cluster-job.yaml.template | kubectl create -f -
+envsubst '${FLINK_IMAGE_NAME} ${FLINK_JOB_PARALLELISM}' < ${CONTAINER_SCRIPTS}/task-manager-deployment.yaml.template | kubectl create -f -
+kubectl wait --for=condition=complete job/flink-job-cluster --timeout=1h
+kubectl cp `kubectl get pods | awk '/task-manager/ {print $1}'`:/cache/${OUTPUT_FILE} ${OUTPUT_VOLUME}/${OUTPUT_FILE}
+
+check_result_hash "WordCount" ${OUTPUT_VOLUME}/${OUTPUT_FILE} "e682ec6622b5e83f2eb614617d5ab2cf"