You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2019/06/05 08:03:05 UTC
[flink] 01/09: [hotfix][e2e] Refactor batch wordcount test to use
different source types
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 16479f4d07327d6991d899d80246c4b48cc390af
Author: Aleksey Pak <al...@ververica.com>
AuthorDate: Mon May 20 14:02:44 2019 +0200
[hotfix][e2e] Refactor batch wordcount test to use different source types
---
flink-end-to-end-tests/run-pre-commit-tests.sh | 6 ++---
.../test-scripts/test_batch_wordcount.sh | 31 ++++++++++++++++++++--
.../test-scripts/test_shaded_hadoop_s3a.sh | 30 ---------------------
.../test-scripts/test_shaded_presto_s3.sh | 30 ---------------------
4 files changed, 32 insertions(+), 65 deletions(-)
diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh
index 3b81b3f..e79542a 100755
--- a/flink-end-to-end-tests/run-pre-commit-tests.sh
+++ b/flink-end-to-end-tests/run-pre-commit-tests.sh
@@ -52,14 +52,14 @@ run_test "State Migration end-to-end test from 1.6" "$END_TO_END_DIR/test-script
run_test "State Evolution end-to-end test" "$END_TO_END_DIR/test-scripts/test_state_evolution.sh"
run_test "Batch Python Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_python_wordcount.sh"
run_test "Streaming Python Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_python_wordcount.sh"
-run_test "Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh"
+run_test "Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh file"
+run_test "Shaded Hadoop S3A end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh hadoop"
+run_test "Shaded Presto S3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh presto"
run_test "Kafka 0.10 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka010.sh"
run_test "Kafka 0.11 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka011.sh"
run_test "Modern Kafka end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka.sh"
run_test "Kinesis end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kinesis.sh"
run_test "class loading end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_classloader.sh"
-run_test "Shaded Hadoop S3A end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_hadoop_s3a.sh"
-run_test "Shaded Presto S3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh"
run_test "Distributed cache end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_distributed_cache_via_blob.sh"
printf "\n[PASS] All tests passed\n"
diff --git a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
index 2c9a175..a5c8222 100755
--- a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
+++ b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
@@ -19,7 +19,34 @@
source "$(dirname "$0")"/common.sh
+INPUT_TYPE=${1:-file}
+case $INPUT_TYPE in
+ (file)
+ INPUT_LOCATION="${TEST_INFRA_DIR}/test-data/words"
+ ;;
+ (hadoop)
+ source "$(dirname "$0")"/common_s3.sh
+ s3_setup hadoop
+ INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
+ ;;
+ (presto)
+ source "$(dirname "$0")"/common_s3.sh
+ s3_setup presto
+ INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
+ ;;
+ (*)
+ echo "Unknown input type $INPUT_TYPE"
+ exit 1
+ ;;
+esac
+
+OUTPUT_LOCATION="${TEST_DATA_DIR}/out/wc_out"
+
+mkdir -p "${TEST_DATA_DIR}"
+
start_cluster
-$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $TEST_INFRA_DIR/test-data/words --output $TEST_DATA_DIR/out/wc_out
-check_result_hash "WordCount" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5"
\ No newline at end of file
+# The test may run against different source types.
+# But the sources should provide the same test data, so the checksum stays the same for all tests.
+"${FLINK_DIR}/bin/flink" run -p 1 "${FLINK_DIR}/examples/batch/WordCount.jar" --input "${INPUT_LOCATION}" --output "${OUTPUT_LOCATION}"
+check_result_hash "WordCount (${INPUT_TYPE})" "${OUTPUT_LOCATION}" "72a690412be8928ba239c2da967328a5"
diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
deleted file mode 100755
index ddbb686..0000000
--- a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# Tests for our shaded/bundled Hadoop S3A file system.
-
-source "$(dirname "$0")"/common.sh
-source "$(dirname "$0")"/common_s3.sh
-
-s3_setup hadoop
-start_cluster
-
-$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out
-
-check_result_hash "WordCountWithShadedS3A" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5"
diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
deleted file mode 100755
index 9ebbb0d..0000000
--- a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-# Tests for our shaded/bundled Hadoop S3A file system.
-
-source "$(dirname "$0")"/common.sh
-source "$(dirname "$0")"/common_s3.sh
-
-s3_setup presto
-start_cluster
-
-$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $S3_TEST_DATA_WORDS_URI --output $TEST_DATA_DIR/out/wc_out
-
-check_result_hash "WordCountWithShadedPrestoS3" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5"