You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/11/16 20:15:17 UTC

[flink] 05/13: [FLINK-10869] [tests] Adjust end-2-end tests to new S3 credentials

This is an automated email from the ASF dual-hosted git repository.

sewen pushed a commit to branch release-1.7
in repository https://gitbox.apache.org/repos/asf/flink.git

commit d3bd21a51bc3811eb898eebcb7c7ba61c8682aee
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Thu Nov 15 23:23:30 2018 +0100

    [FLINK-10869] [tests] Adjust end-2-end tests to new S3 credentials
---
 flink-end-to-end-tests/test-scripts/common_s3.sh   | 60 +++++++++++-----------
 .../test-scripts/test_shaded_hadoop_s3a.sh         |  4 +-
 .../test-scripts/test_shaded_presto_s3.sh          |  4 +-
 .../test-scripts/test_streaming_file_sink.sh       |  2 +-
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/flink-end-to-end-tests/test-scripts/common_s3.sh b/flink-end-to-end-tests/test-scripts/common_s3.sh
index 5c16bb7..784d573 100644
--- a/flink-end-to-end-tests/test-scripts/common_s3.sh
+++ b/flink-end-to-end-tests/test-scripts/common_s3.sh
@@ -17,30 +17,30 @@
 # limitations under the License.
 ################################################################################
 
-if [[ -z "$ARTIFACTS_AWS_BUCKET" ]]; then
+if [[ -z "$IT_CASE_S3_BUCKET" ]]; then
     echo "Did not find AWS environment variables, NOT running the e2e test."
     exit 0
 else
-    echo "Found AWS bucket $ARTIFACTS_AWS_BUCKET, running the e2e test."
+    echo "Found AWS bucket $IT_CASE_S3_BUCKET, running the e2e test."
 fi
 
-if [[ -z "$ARTIFACTS_AWS_ACCESS_KEY" ]]; then
+if [[ -z "$IT_CASE_S3_ACCESS_KEY" ]]; then
     echo "Did not find AWS environment variables, NOT running the e2e test."
     exit 0
 else
-    echo "Found AWS access key $ARTIFACTS_AWS_ACCESS_KEY, running the e2e test."
+    echo "Found AWS access key $IT_CASE_S3_ACCESS_KEY, running the e2e test."
 fi
 
-if [[ -z "$ARTIFACTS_AWS_SECRET_KEY" ]]; then
+if [[ -z "$IT_CASE_S3_SECRET_KEY" ]]; then
     echo "Did not find AWS environment variables, NOT running the e2e test."
     exit 0
 else
-    echo "Found AWS secret key $ARTIFACTS_AWS_SECRET_KEY, running the e2e test."
+    echo "Found AWS secret key $IT_CASE_S3_SECRET_KEY, running the e2e test."
 fi
 
-AWS_REGION="${AWS_REGION:-eu-west-1}"
-AWS_ACCESS_KEY=$ARTIFACTS_AWS_ACCESS_KEY
-AWS_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY
+AWS_REGION="${AWS_REGION:-us-east-1}"
+AWS_ACCESS_KEY=$IT_CASE_S3_ACCESS_KEY
+AWS_SECRET_KEY=$IT_CASE_S3_SECRET_KEY
 
 s3util="java -jar ${END_TO_END_DIR}/flink-e2e-test-utils/target/S3UtilProgram.jar"
 
@@ -49,8 +49,8 @@ s3util="java -jar ${END_TO_END_DIR}/flink-e2e-test-utils/target/S3UtilProgram.ja
 #
 # Globals:
 #   FLINK_DIR
-#   ARTIFACTS_AWS_ACCESS_KEY
-#   ARTIFACTS_AWS_SECRET_KEY
+#   IT_CASE_S3_ACCESS_KEY
+#   IT_CASE_S3_SECRET_KEY
 # Arguments:
 #   None
 # Returns:
@@ -68,8 +68,8 @@ function s3_setup {
   trap s3_cleanup EXIT
 
   cp $FLINK_DIR/opt/flink-s3-fs-hadoop-*.jar $FLINK_DIR/lib/
-  echo "s3.access-key: $ARTIFACTS_AWS_ACCESS_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
-  echo "s3.secret-key: $ARTIFACTS_AWS_SECRET_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
+  echo "s3.access-key: $IT_CASE_S3_ACCESS_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
+  echo "s3.secret-key: $IT_CASE_S3_SECRET_KEY" >> "$FLINK_DIR/conf/flink-conf.yaml"
 }
 
 s3_setup
@@ -78,7 +78,7 @@ s3_setup
 # List s3 objects by full path prefix.
 #
 # Globals:
-#   ARTIFACTS_AWS_BUCKET
+#   IT_CASE_S3_BUCKET
 # Arguments:
 #   $1 - s3 full path key prefix
 # Returns:
@@ -86,14 +86,14 @@ s3_setup
 ###################################
 function s3_list {
   AWS_REGION=$AWS_REGION \
-  ${s3util} --action listByFullPathPrefix --s3prefix "$1" --bucket $ARTIFACTS_AWS_BUCKET
+  ${s3util} --action listByFullPathPrefix --s3prefix "$1" --bucket $IT_CASE_S3_BUCKET
 }
 
 ###################################
 # Download s3 object.
 #
 # Globals:
-#   ARTIFACTS_AWS_BUCKET
+#   IT_CASE_S3_BUCKET
 # Arguments:
 #   $1 - local path to save file
 #   $2 - s3 object key
@@ -102,14 +102,14 @@ function s3_list {
 ###################################
 function s3_get {
   AWS_REGION=$AWS_REGION \
-  ${s3util} --action downloadFile --localFile "$1" --s3file "$2" --bucket $ARTIFACTS_AWS_BUCKET
+  ${s3util} --action downloadFile --localFile "$1" --s3file "$2" --bucket $IT_CASE_S3_BUCKET
 }
 
 ###################################
 # Download s3 objects to folder by full path prefix.
 #
 # Globals:
-#   ARTIFACTS_AWS_BUCKET
+#   IT_CASE_S3_BUCKET
 # Arguments:
 #   $1 - local path to save folder with files
 #   $2 - s3 key full path prefix
@@ -121,14 +121,14 @@ function s3_get_by_full_path_and_filename_prefix {
   local file_prefix="${3-}"
   AWS_REGION=$AWS_REGION \
   ${s3util} --action downloadByFullPathAndFileNamePrefix \
-    --localFolder "$1" --s3prefix "$2" --s3filePrefix "${file_prefix}" --bucket $ARTIFACTS_AWS_BUCKET
+    --localFolder "$1" --s3prefix "$2" --s3filePrefix "${file_prefix}" --bucket $IT_CASE_S3_BUCKET
 }
 
 ###################################
 # Upload file to s3 object.
 #
 # Globals:
-#   ARTIFACTS_AWS_BUCKET
+#   IT_CASE_S3_BUCKET
 # Arguments:
 #   $1 - local file to upload
 #   $2 - s3 bucket
@@ -144,8 +144,8 @@ function s3_put {
   contentType="application/octet-stream"
   dateValue=`date -R`
   stringToSign="PUT\n\n${contentType}\n${dateValue}\n${resource}"
-  s3Key=$ARTIFACTS_AWS_ACCESS_KEY
-  s3Secret=$ARTIFACTS_AWS_SECRET_KEY
+  s3Key=$IT_CASE_S3_ACCESS_KEY
+  s3Secret=$IT_CASE_S3_SECRET_KEY
   signature=`echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64`
   curl -X PUT -T "${local_file}" \
     -H "Host: ${bucket}.s3.amazonaws.com" \
@@ -174,8 +174,8 @@ function s3_delete {
   contentType="application/octet-stream"
   dateValue=`date -R`
   stringToSign="DELETE\n\n${contentType}\n${dateValue}\n${resource}"
-  s3Key=$ARTIFACTS_AWS_ACCESS_KEY
-  s3Secret=$ARTIFACTS_AWS_SECRET_KEY
+  s3Key=$IT_CASE_S3_ACCESS_KEY
+  s3Secret=$IT_CASE_S3_SECRET_KEY
   signature=`echo -en ${stringToSign} | openssl sha1 -hmac ${s3Secret} -binary | base64`
   curl -X DELETE \
     -H "Host: ${bucket}.s3.amazonaws.com" \
@@ -189,7 +189,7 @@ function s3_delete {
 # Delete s3 objects by full path prefix.
 #
 # Globals:
-#   ARTIFACTS_AWS_BUCKET
+#   IT_CASE_S3_BUCKET
 # Arguments:
 #   $1 - s3 key full path prefix
 # Returns:
@@ -197,7 +197,7 @@ function s3_delete {
 ###################################
 function s3_delete_by_full_path_prefix {
   AWS_REGION=$AWS_REGION \
-  ${s3util} --action deleteByFullPathPrefix --s3prefix "$1" --bucket $ARTIFACTS_AWS_BUCKET
+  ${s3util} --action deleteByFullPathPrefix --s3prefix "$1" --bucket $IT_CASE_S3_BUCKET
 }
 
 ###################################
@@ -206,7 +206,7 @@ function s3_delete_by_full_path_prefix {
 # because SQL is used to query the s3 object.
 #
 # Globals:
-#   ARTIFACTS_AWS_BUCKET
+#   IT_CASE_S3_BUCKET
 # Arguments:
 #   $1 - s3 file object key
 #   $2 - s3 bucket
@@ -215,7 +215,7 @@ function s3_delete_by_full_path_prefix {
 ###################################
 function s3_get_number_of_lines_in_file {
   AWS_REGION=$AWS_REGION \
-  ${s3util} --action numberOfLinesInFile --s3file "$1" --bucket $ARTIFACTS_AWS_BUCKET
+  ${s3util} --action numberOfLinesInFile --s3file "$1" --bucket $IT_CASE_S3_BUCKET
 }
 
 ###################################
@@ -224,7 +224,7 @@ function s3_get_number_of_lines_in_file {
 # because SQL is used to query the s3 objects.
 #
 # Globals:
-#   ARTIFACTS_AWS_BUCKET
+#   IT_CASE_S3_BUCKET
 # Arguments:
 #   $1 - s3 key prefix
 #   $2 - s3 bucket
@@ -236,5 +236,5 @@ function s3_get_number_of_lines_by_prefix {
   local file_prefix="${3-}"
   AWS_REGION=$AWS_REGION \
   ${s3util} --action numberOfLinesInFilesWithFullAndNamePrefix \
-    --s3prefix "$1" --s3filePrefix "${file_prefix}" --bucket $ARTIFACTS_AWS_BUCKET
+    --s3prefix "$1" --s3filePrefix "${file_prefix}" --bucket $IT_CASE_S3_BUCKET
 }
diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
index 3d83867..e6ae992 100755
--- a/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
+++ b/flink-end-to-end-tests/test-scripts/test_shaded_hadoop_s3a.sh
@@ -22,10 +22,10 @@
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/common_s3.sh
 
-s3_put $TEST_INFRA_DIR/test-data/words $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-s3a
+s3_put $TEST_INFRA_DIR/test-data/words $IT_CASE_S3_BUCKET temp/flink-end-to-end-test-shaded-s3a
 # make sure we delete the file at the end
 function shaded_s3a_cleanup {
-  s3_delete $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-s3a
+  s3_delete $IT_CASE_S3_BUCKET temp/flink-end-to-end-test-shaded-s3a
 }
 trap shaded_s3a_cleanup EXIT
 
diff --git a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
index bd33b41..0421c84 100755
--- a/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
+++ b/flink-end-to-end-tests/test-scripts/test_shaded_presto_s3.sh
@@ -22,10 +22,10 @@
 source "$(dirname "$0")"/common.sh
 source "$(dirname "$0")"/common_s3.sh
 
-s3_put $TEST_INFRA_DIR/test-data/words $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-presto-s3
+s3_put $TEST_INFRA_DIR/test-data/words $IT_CASE_S3_BUCKET temp/flink-end-to-end-test-shaded-presto-s3
 # make sure we delete the file at the end
 function shaded_presto_s3_cleanup {
-  s3_delete $ARTIFACTS_AWS_BUCKET flink-end-to-end-test-shaded-presto-s3
+  s3_delete $IT_CASE_S3_BUCKET temp/flink-end-to-end-test-shaded-presto-s3
 }
 trap shaded_presto_s3_cleanup EXIT
 
diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh b/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh
index 6c8d0b8..7a469aa 100755
--- a/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh
+++ b/flink-end-to-end-tests/test-scripts/test_streaming_file_sink.sh
@@ -26,7 +26,7 @@ set_conf_ssl "mutual"
 
 OUT=out
 OUTPUT_PATH="$TEST_DATA_DIR/$OUT"
-S3_OUTPUT_PATH="s3://$ARTIFACTS_AWS_BUCKET/$OUT"
+S3_OUTPUT_PATH="s3://$IT_CASE_S3_BUCKET/temp/$OUT"
 
 mkdir -p $OUTPUT_PATH