You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by pn...@apache.org on 2019/06/13 10:09:03 UTC
[flink] 10/10: [FLINK-12143][e2e] Change end-to-end tests with file
system to use plugins mechanism
This is an automated email from the ASF dual-hosted git repository.
pnowojski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 9e6ff81e22d6f5f04abb50ca1aea84fd2542bf9d
Author: Aleksey Pak <al...@ververica.com>
AuthorDate: Thu Jun 6 12:42:21 2019 +0200
[FLINK-12143][e2e] Change end-to-end tests with file system to use plugins mechanism
---
.../test-scripts/common_dummy_fs.sh | 23 ++++++++++++++++++++++
flink-end-to-end-tests/test-scripts/common_s3.sh | 2 +-
.../test-scripts/test_azure_fs.sh | 2 +-
.../test-scripts/test_batch_wordcount.sh | 3 ++-
.../test-scripts/test_docker_embedded_job.sh | 3 ++-
.../test-scripts/test_yarn_kerberos_docker.sh | 3 ++-
6 files changed, 31 insertions(+), 5 deletions(-)
diff --git a/flink-end-to-end-tests/test-scripts/common_dummy_fs.sh b/flink-end-to-end-tests/test-scripts/common_dummy_fs.sh
new file mode 100644
index 0000000..bd6e756
--- /dev/null
+++ b/flink-end-to-end-tests/test-scripts/common_dummy_fs.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+function dummy_fs_setup() {
+ mkdir -p "$FLINK_DIR/plugins/dummy-fs"
+ cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/plugins/dummy-fs/"
+}
diff --git a/flink-end-to-end-tests/test-scripts/common_s3.sh b/flink-end-to-end-tests/test-scripts/common_s3.sh
index 6536a88..b423977 100644
--- a/flink-end-to-end-tests/test-scripts/common_s3.sh
+++ b/flink-end-to-end-tests/test-scripts/common_s3.sh
@@ -63,7 +63,7 @@ s3util="java -jar ${END_TO_END_DIR}/flink-e2e-test-utils/target/S3UtilProgram.ja
# None
###################################
function s3_setup {
- add_optional_lib "s3-fs-$1"
+ add_optional_plugin "s3-fs-$1"
set_config_key "s3.access-key" "$IT_CASE_S3_ACCESS_KEY"
set_config_key "s3.secret-key" "$IT_CASE_S3_SECRET_KEY"
}
diff --git a/flink-end-to-end-tests/test-scripts/test_azure_fs.sh b/flink-end-to-end-tests/test-scripts/test_azure_fs.sh
index f68c6d8..b200e49 100755
--- a/flink-end-to-end-tests/test-scripts/test_azure_fs.sh
+++ b/flink-end-to-end-tests/test-scripts/test_azure_fs.sh
@@ -61,7 +61,7 @@ AZURE_TEST_DATA_WORDS_URI="wasbs://$IT_CASE_AZURE_CONTAINER@$IT_CASE_AZURE_ACCOU
function azure_setup {
echo "Copying flink azure jars and writing out configs"
- add_optional_lib "azure-fs-hadoop"
+ add_optional_plugin "azure-fs-hadoop"
set_config_key "fs.azure.account.key.$IT_CASE_AZURE_ACCOUNT.blob.core.windows.net" "$IT_CASE_AZURE_ACCESS_KEY"
}
diff --git a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
index 3b8400b..53bd585 100755
--- a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
+++ b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
@@ -35,7 +35,8 @@ case $INPUT_TYPE in
INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
;;
(dummy-fs)
- cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/lib/"
+ source "$(dirname "$0")"/common_dummy_fs.sh
+ dummy_fs_setup
INPUT_LOCATION="dummy://localhost/words"
;;
(*)
diff --git a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
index dabb3ce..4b87d57 100755
--- a/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
+++ b/flink-end-to-end-tests/test-scripts/test_docker_embedded_job.sh
@@ -37,7 +37,8 @@ case $INPUT_TYPE in
INPUT_LOCATION=${INPUT_PATH}/words
;;
(dummy-fs)
- cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/lib/"
+ source "$(dirname "$0")"/common_dummy_fs.sh
+ dummy_fs_setup
INPUT_LOCATION="dummy://localhost/words"
;;
(*)
diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
index 5199b2a..6f66236 100755
--- a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
+++ b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
@@ -43,7 +43,8 @@ case $INPUT_TYPE in
EXPECTED_RESULT_LOG_CONTAINS=("consummation,1" "of,14" "calamity,1")
;;
(dummy-fs)
- cp "${END_TO_END_DIR}/flink-plugins-test/target/flink-dummy-fs.jar" "${FLINK_DIR}/lib/"
+ source "$(dirname "$0")"/common_dummy_fs.sh
+ dummy_fs_setup
INPUT_ARGS="--input dummy://localhost/words"
EXPECTED_RESULT_LOG_CONTAINS=("my,1" "dear,2" "world,2")
;;