You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2019/12/21 18:07:27 UTC
[flink] 02/02: [FLINK-10938][e2e] Add e2e test for natively running
flink session cluster on kubernetes
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 7d8e9eaf88c3a7903b3cf6ed4d41a06d385b6d66
Author: wangyang0918 <da...@alibaba-inc.com>
AuthorDate: Fri Dec 13 20:23:34 2019 +0800
[FLINK-10938][e2e] Add e2e test for natively running flink session cluster on kubernetes
This closes #10602.
---
flink-end-to-end-tests/run-nightly-tests.sh | 1 +
.../test-scripts/common_kubernetes.sh | 3 +-
.../test-scripts/test_kubernetes_session.sh | 58 ++++++++++++++++++++++
tools/travis/splits/split_container.sh | 1 +
4 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index 4fda602..7c8d181 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -112,6 +112,7 @@ run_test "Running HA per-job cluster (rocks, incremental) end-to-end test" "$END
################################################################################
run_test "Run Kubernetes test" "$END_TO_END_DIR/test-scripts/test_kubernetes_embedded_job.sh"
+run_test "Run kubernetes session test" "$END_TO_END_DIR/test-scripts/test_kubernetes_session.sh"
################################################################################
# Miscellaneous
diff --git a/flink-end-to-end-tests/test-scripts/common_kubernetes.sh b/flink-end-to-end-tests/test-scripts/common_kubernetes.sh
index d651c01..d3509cd 100755
--- a/flink-end-to-end-tests/test-scripts/common_kubernetes.sh
+++ b/flink-end-to-end-tests/test-scripts/common_kubernetes.sh
@@ -45,8 +45,7 @@ function start_kubernetes {
echo "Minikube not running. Could not start minikube. Aborting..."
exit 1
fi
+ eval $(minikube docker-env)
}
on_exit cleanup
-
-eval $(minikube docker-env)
diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_session.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_session.sh
new file mode 100755
index 0000000..845e8a2
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_session.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+source "$(dirname "$0")"/common_kubernetes.sh
+
+CLUSTER_ROLE_BINDING="flink-role-binding-default"
+CLUSTER_ID="flink-native-k8s-session-1"
+FLINK_IMAGE_NAME="test_kubernetes_session"
+LOCAL_OUTPUT_PATH="${TEST_DATA_DIR}/out/wc_out"
+OUTPUT_PATH="/tmp/wc_out"
+ARGS="--output ${OUTPUT_PATH}"
+
+function cleanup {
+ kubectl delete service ${CLUSTER_ID}
+ kubectl delete clusterrolebinding ${CLUSTER_ROLE_BINDING}
+}
+
+start_kubernetes
+
+cd "$DOCKER_MODULE_DIR"
+# Build a Flink image without any user jars
+./build.sh --from-local-dist --job-artifacts ${TEST_INFRA_DIR}/test-data/words --image-name ${FLINK_IMAGE_NAME}
+
+kubectl create clusterrolebinding ${CLUSTER_ROLE_BINDING} --clusterrole=edit --serviceaccount=default:default --namespace=default
+
+mkdir -p "$(dirname $LOCAL_OUTPUT_PATH)"
+
+# Set the memory and cpu smaller than default, so that the jobmanager and taskmanager pods could be allocated in minikube.
+"$FLINK_DIR"/bin/kubernetes-session.sh -Dkubernetes.cluster-id=${CLUSTER_ID} \
+ -Dkubernetes.container.image=${FLINK_IMAGE_NAME} \
+ -Djobmanager.heap.size=512m \
+ -Dcontainerized.heap-cutoff-min=100 \
+ -Dkubernetes.jobmanager.cpu=0.5 \
+ -Dkubernetes.taskmanager.cpu=0.5
+
+"$FLINK_DIR"/bin/flink run -e kubernetes-session \
+ -Dkubernetes.cluster-id=${CLUSTER_ID} \
+ ${FLINK_DIR}/examples/batch/WordCount.jar ${ARGS}
+
+kubectl cp `kubectl get pods | awk '/taskmanager/ {print $1}'`:${OUTPUT_PATH} ${LOCAL_OUTPUT_PATH}
+
+check_result_hash "WordCount" "${LOCAL_OUTPUT_PATH}" "${RESULT_HASH}"
diff --git a/tools/travis/splits/split_container.sh b/tools/travis/splits/split_container.sh
index 587f397..6ee4368 100755
--- a/tools/travis/splits/split_container.sh
+++ b/tools/travis/splits/split_container.sh
@@ -47,6 +47,7 @@ run_test "Wordcount on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scr
run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
run_test "Run kubernetes test" "$END_TO_END_DIR/test-scripts/test_kubernetes_embedded_job.sh"
+run_test "Run kubernetes session test" "$END_TO_END_DIR/test-scripts/test_kubernetes_session.sh"
printf "\n[PASS] All tests passed\n"
exit 0