[flink] branch master updated: [FLINK-13044][s3][fs] Fix handling of relocated amazon classes

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 5e5f5be  [FLINK-13044][s3][fs] Fix handling of relocated amazon classes
5e5f5be is described below

commit 5e5f5bec72d30397253fc1f11105ca385ca8681f
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Aug 1 12:40:21 2019 +0200

    [FLINK-13044][s3][fs] Fix handling of relocated amazon classes
    
    To provide credentials to S3 users may configure a credentials provider. For providers from amazon (which are relocated) we allow users to configure the original class name, and relocate it manually in the S3 filesystem factories.
    
    The factories however
    - used the wrong shading pattern
    - could not correctly detect providers with a "com.amazon*" prefix, as the String constant containing this prefix was also being relocated.
    
    This commit obfuscates the constant to prevent relocations, fixes the shading patterns, and sets up e2e tests to cover this case.
---
 flink-end-to-end-tests/test-scripts/common_s3.sh                    | 6 ++++++
 flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh         | 5 +++++
 .../main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java | 6 ++++--
 .../java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java   | 2 +-
 .../main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java | 6 ++++--
 .../java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java   | 2 +-
 tools/travis/splits/split_misc.sh                                   | 2 ++
 7 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/flink-end-to-end-tests/test-scripts/common_s3.sh b/flink-end-to-end-tests/test-scripts/common_s3.sh
index b423977..aeb9426 100644
--- a/flink-end-to-end-tests/test-scripts/common_s3.sh
+++ b/flink-end-to-end-tests/test-scripts/common_s3.sh
@@ -68,6 +68,12 @@ function s3_setup {
   set_config_key "s3.secret-key" "$IT_CASE_S3_SECRET_KEY"
 }
 
+function s3_setup_with_provider {
+  add_optional_plugin "s3-fs-$1"
+  # reads (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+  set_config_key "$2" "com.amazonaws.auth.EnvironmentVariableCredentialsProvider"
+}
+
 ###################################
 # List s3 objects by full path prefix.
 #
diff --git a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
index 53bd585..2e90e0b 100755
--- a/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
+++ b/flink-end-to-end-tests/test-scripts/test_batch_wordcount.sh
@@ -29,6 +29,11 @@ case $INPUT_TYPE in
         s3_setup hadoop
         INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
     ;;
+    (hadoop_with_provider)
+        source "$(dirname "$0")"/common_s3.sh
+        s3_setup_with_provider hadoop "fs.s3a.aws.credentials.provider"
+        INPUT_LOCATION="${S3_TEST_DATA_WORDS_URI}"
+    ;;
     (presto)
         source "$(dirname "$0")"/common_s3.sh
         s3_setup presto
diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java
index 6cad051..7a6ac0a 100644
--- a/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java
+++ b/flink-filesystems/flink-s3-fs-hadoop/src/main/java/org/apache/flink/fs/s3hadoop/S3FileSystemFactory.java
@@ -41,11 +41,13 @@ public class S3FileSystemFactory extends AbstractS3FileSystemFactory {
 
 	private static final Logger LOG = LoggerFactory.getLogger(S3FileSystemFactory.class);
 
-	private static final Set<String> PACKAGE_PREFIXES_TO_SHADE = Collections.singleton("com.amazonaws.");
+	// intentionally obfuscated to prevent relocations by the shade-plugin
+	private static final Set<String> PACKAGE_PREFIXES_TO_SHADE = Collections.singleton("com.UNSHADE.".replace("UNSHADE", "amazonaws"));
 
 	private static final Set<String> CONFIG_KEYS_TO_SHADE = Collections.singleton("fs.s3a.aws.credentials.provider");
 
-	private static final String FLINK_SHADING_PREFIX = "org.apache.flink.fs.s3hadoop.shaded.";
+	// keep this in sync with the relocation pattern applied to com.amazon in the shade-plugin configuration
+	private static final String FLINK_SHADING_PREFIX = "org.apache.flink.fs.s3base.shaded.";
 
 	private static final String[] FLINK_CONFIG_PREFIXES = { "s3.", "s3a.", "fs.s3a." };
 
diff --git a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java
index 57500f3..e7ca6b4 100644
--- a/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java
+++ b/flink-filesystems/flink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemTest.java
@@ -39,7 +39,7 @@ public class HadoopS3FileSystemTest {
 		configLoader.setFlinkConfig(conf);
 
 		org.apache.hadoop.conf.Configuration hadoopConfig = configLoader.getOrLoadHadoopConfig();
-		assertEquals("org.apache.flink.fs.s3hadoop.shaded.com.amazonaws.auth.ContainerCredentialsProvider",
+		assertEquals("org.apache.flink.fs.s3base.shaded.com.amazonaws.auth.ContainerCredentialsProvider",
 			hadoopConfig.get("fs.s3a.aws.credentials.provider"));
 	}
 
diff --git a/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java
index 5a1ffee..df27b12 100644
--- a/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java
+++ b/flink-filesystems/flink-s3-fs-presto/src/main/java/org/apache/flink/fs/s3presto/S3FileSystemFactory.java
@@ -39,11 +39,13 @@ import java.util.Set;
  */
 public class S3FileSystemFactory extends AbstractS3FileSystemFactory {
 
-	private static final Set<String> PACKAGE_PREFIXES_TO_SHADE = Collections.singleton("com.amazonaws.");
+	// intentionally obfuscated to prevent relocations by the shade-plugin
+	private static final Set<String> PACKAGE_PREFIXES_TO_SHADE = Collections.singleton("com.UNSHADE.".replace("UNSHADE", "amazonaws"));
 
 	private static final Set<String> CONFIG_KEYS_TO_SHADE = Collections.singleton("presto.s3.credentials-provider");
 
-	private static final String FLINK_SHADING_PREFIX = "org.apache.flink.fs.s3presto.shaded.";
+	// keep this in sync with the relocation pattern applied to com.amazon in the shade-plugin configuration
+	private static final String FLINK_SHADING_PREFIX = "org.apache.flink.fs.s3base.shaded.";
 
 	private static final String[] FLINK_CONFIG_PREFIXES = { "s3.", "presto.s3." };
 
diff --git a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java
index f3117a2..f103e95 100644
--- a/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java
+++ b/flink-filesystems/flink-s3-fs-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemTest.java
@@ -86,7 +86,7 @@ public class PrestoS3FileSystemTest {
 		configLoader.setFlinkConfig(conf);
 
 		org.apache.hadoop.conf.Configuration hadoopConfig = configLoader.getOrLoadHadoopConfig();
-		assertEquals("org.apache.flink.fs.s3presto.shaded.com.amazonaws.auth.ContainerCredentialsProvider",
+		assertEquals("org.apache.flink.fs.s3base.shaded.com.amazonaws.auth.ContainerCredentialsProvider",
 			hadoopConfig.get("presto.s3.credentials-provider"));
 	}
 
diff --git a/tools/travis/splits/split_misc.sh b/tools/travis/splits/split_misc.sh
index 0171745..9bc5a27 100755
--- a/tools/travis/splits/split_misc.sh
+++ b/tools/travis/splits/split_misc.sh
@@ -76,5 +76,7 @@ run_test "SQL Client end-to-end test for modern Kafka" "$END_TO_END_DIR/test-scr
 
 run_test "Dependency shading of table modules test" "$END_TO_END_DIR/test-scripts/test_table_shaded_dependencies.sh"
 
+run_test "Shaded Hadoop S3A with credentials provider end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh hadoop_with_provider"
+
 printf "\n[PASS] All tests passed\n"
 exit 0