You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by kk...@apache.org on 2020/04/30 20:10:54 UTC
[flink] 10/13: [FLINK-16661] Add YARN application end-to-end test
This is an automated email from the ASF dual-hosted git repository.
kkloudas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 59f304029fda3bb905ddfba9212743f60eb1ec37
Author: Aljoscha Krettek <al...@apache.org>
AuthorDate: Fri Apr 24 18:46:10 2020 +0200
[FLINK-16661] Add YARN application end-to-end test
This renames the existing test_yarn_kerberos_docker.sh to
test_yarn_job_kerberos_docker.sh to reflect that it is a per-job test
and adds a new test based on that for YARN application mode.
---
flink-end-to-end-tests/run-nightly-tests.sh | 8 ++--
.../test-scripts/common_yarn_docker.sh | 46 +++++++++++++++++++++-
...sh => test_yarn_application_kerberos_docker.sh} | 35 ++++++++++++----
..._docker.sh => test_yarn_job_kerberos_docker.sh} | 0
4 files changed, 76 insertions(+), 13 deletions(-)
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index 04040ee..e8c3d33 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -131,9 +131,11 @@ if [[ ${PROFILE} != *"jdk11"* ]]; then
if [[ `uname -i` != 'aarch64' ]]; then
# Hadoop YARN deosn't support aarch64 at this moment. See: https://issues.apache.org/jira/browse/HADOOP-16723
- run_test "Running Kerberized YARN on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh"
- run_test "Running Kerberized YARN on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_kerberos_docker.sh dummy-fs"
- if [[ $PROFILE == *"include-hadoop"* ]]; then
+ run_test "Running Kerberized YARN per-job on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_job_kerberos_docker.sh"
+ run_test "Running Kerberized YARN per-job on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_job_kerberos_docker.sh dummy-fs"
+ run_test "Running Kerberized YARN application on Docker test (default input)" "$END_TO_END_DIR/test-scripts/test_yarn_application_kerberos_docker.sh"
+ run_test "Running Kerberized YARN application on Docker test (custom fs plugin)" "$END_TO_END_DIR/test-scripts/test_yarn_application_kerberos_docker.sh dummy-fs"
+ if [[ $PROFILE == *"include-hadoop"* ]]; then
run_test "Run Mesos WordCount test" "$END_TO_END_DIR/test-scripts/test_mesos_wordcount.sh"
run_test "Run Mesos multiple submission test" "$END_TO_END_DIR/test-scripts/test_mesos_multiple_submissions.sh"
fi
diff --git a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
index 34d6ef4..009fd7e 100755
--- a/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
+++ b/flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
@@ -147,7 +147,7 @@ function debug_copy_and_show_logs {
echo "$f:"
cat $f
done
-
+
echo "Docker logs:"
docker logs master
@@ -155,12 +155,54 @@ function debug_copy_and_show_logs {
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list -appStates ALL" | grep "Flink" | grep "cluster" | awk '{print \$1}'`
-
+
echo "Application ID: $application_id"
docker exec master bash -c "yarn logs -applicationId $application_id"
docker exec master bash -c "kdestroy"
}
+# expects only one application to be running and waits until this one is in
+# final state SUCCEEDED
+function wait_for_single_yarn_application {
+
+ docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user"
+
+ # find our application ID
+ docker exec master bash -c "yarn application -list -appStates ALL"
+ application_id=$(docker exec master bash -c "yarn application -list -appStates ALL" | grep "Flink Application" | awk '{print $1}')
+
+ echo "Application ID: $application_id"
+
+ # wait for the application to finish succesfully
+ start_time=$(date +%s)
+ application_state="UNDEFINED"
+ while [[ $application_state != "FINISHED" ]]; do
+ current_time=$(date +%s)
+ time_diff=$((current_time - start_time))
+
+ if [[ $time_diff -ge $MAX_RETRY_SECONDS ]]; then
+ echo "Application $application_id is in state $application_state and we have waited too long, quitting..."
+ exit 1
+ else
+ echo "Application $application_id is in state $application_state. We have been waiting for $time_diff seconds, looping ..."
+ sleep 1
+ fi
+
+ application_state=$(docker exec master bash -c "yarn application -status $application_id" | grep "\sState" | sed 's/.*State : \(\w*\)/\1/')
+ done
+
+ final_application_state=$(docker exec master bash -c "yarn application -status $application_id" | grep "\sFinal-State" | sed 's/.*Final-State : \(\w*\)/\1/')
+
+ echo "Final Application State: $final_application_state"
+
+ if [[ $final_application_state != "SUCCEEDED" ]]; then
+ echo "Running the Flink Application failed. 😞"
+ exit 1
+ fi
+
+ docker exec master bash -c "kdestroy"
+}
+
function get_output {
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab hadoop-user"
docker exec master bash -c "hdfs dfs -ls $1"
diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh b/flink-end-to-end-tests/test-scripts/test_yarn_application_kerberos_docker.sh
similarity index 76%
copy from flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
copy to flink-end-to-end-tests/test-scripts/test_yarn_application_kerberos_docker.sh
index a86860f..5309507 100755
--- a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
+++ b/flink-end-to-end-tests/test-scripts/test_yarn_application_kerberos_docker.sh
@@ -17,7 +17,10 @@
# limitations under the License.
################################################################################
+# shellcheck source=common.sh
source "$(dirname "$0")"/common.sh
+
+# shellcheck source=common_yarn_docker.sh
source "$(dirname "$0")"/common_yarn_docker.sh
# Configure Flink dir before making tarball.
@@ -29,6 +32,7 @@ case $INPUT_TYPE in
EXPECTED_RESULT_LOG_CONTAINS=("consummation,1" "of,14" "calamity,1")
;;
(dummy-fs)
+ # shellcheck source=common_dummy_fs.sh
source "$(dirname "$0")"/common_dummy_fs.sh
dummy_fs_setup
INPUT_ARGS="--input dummy://localhost/words --input anotherDummy://localhost/words"
@@ -49,18 +53,29 @@ OUTPUT_PATH=hdfs:///user/hadoop-user/wc-out-$RANDOM
# it's important to run this with higher parallelism, otherwise we might risk that
# JM and TM are on the same YARN node and that we therefore don't test the keytab shipping
if docker exec master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && \
- /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster -ys 1 -ytm 1000 -yjm 1000 -p 3 \
- -yD taskmanager.memory.jvm-metaspace.size=128m \
+ /home/hadoop-user/$FLINK_DIRNAME/bin/flink run-application \
+ -t yarn-application \
+ -Dtaskmanager.numberOfTaskSlots=1 \
+ -Dtaskmanager.memory.process.size=1000m \
+ -Djobmanager.memory.process.size=1000m \
+ -Dparallelism.default=3 \
+ -Dtaskmanager.memory.jvm-metaspace.size=128m \
/home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar $INPUT_ARGS --output $OUTPUT_PATH";
then
- OUTPUT=$(get_output "$OUTPUT_PATH/*")
- echo "$OUTPUT"
+ echo "Flink YARN Application submitted."
else
echo "Running the job failed."
exit 1
fi
-for expected_result in ${EXPECTED_RESULT_LOG_CONTAINS[@]}; do
+wait_for_single_yarn_application
+
+# now we should have the application output ready
+
+OUTPUT=$(get_output "$OUTPUT_PATH/*")
+echo "$OUTPUT"
+
+for expected_result in "${EXPECTED_RESULT_LOG_CONTAINS[@]}"; do
if [[ ! "$OUTPUT" =~ $expected_result ]]; then
echo "Output does not contain '$expected_result' as required"
exit 1
@@ -71,9 +86,13 @@ echo "Running Job without configured keytab, the exception you see below is expe
docker exec master bash -c "echo \"\" > /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
# verify that it doesn't work if we don't configure a keytab
docker exec master bash -c "export HADOOP_CLASSPATH=\`hadoop classpath\` && \
- /home/hadoop-user/$FLINK_DIRNAME/bin/flink run \
- -m yarn-cluster -ys 1 -ytm 1000 -yjm 1000 -p 3 \
- -yD taskmanager.memory.jvm-metaspace.size=128m \
+ /home/hadoop-user/$FLINK_DIRNAME/bin/flink run-application \
+ -t yarn-application \
+ -Dtaskmanager.numberOfTaskSlots=1 \
+ -Dtaskmanager.memory.process.size=1000m \
+ -Djobmanager.memory.process.size=1000m \
+ -Dparallelism.default=3 \
+ -Dtaskmanager.memory.jvm-metaspace.size=128m \
/home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output $OUTPUT_PATH" > stderrAndstdoutFile 2>&1
OUTPUT=$(cat stderrAndstdoutFile)
rm stderrAndstdoutFile
diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh b/flink-end-to-end-tests/test-scripts/test_yarn_job_kerberos_docker.sh
similarity index 100%
rename from flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
rename to flink-end-to-end-tests/test-scripts/test_yarn_job_kerberos_docker.sh