You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tr...@apache.org on 2019/05/28 15:27:24 UTC
[flink] 03/04: [FLINK-12115][fs] Decrease dependency footprint of
flink-azure-fs-hadoop
This is an automated email from the ASF dual-hosted git repository.
trohrmann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 09dae1ff4380b8584c9fc52b8aac32edc96eaa2e
Author: Till Rohrmann <tr...@apache.org>
AuthorDate: Fri May 24 15:32:12 2019 +0200
[FLINK-12115][fs] Decrease dependency footprint of flink-azure-fs-hadoop
---
flink-filesystems/flink-azure-fs-hadoop/pom.xml | 92 ++++++++++++++++++-------
1 file changed, 67 insertions(+), 25 deletions(-)
diff --git a/flink-filesystems/flink-azure-fs-hadoop/pom.xml b/flink-filesystems/flink-azure-fs-hadoop/pom.xml
index 37567ce..781e851 100644
--- a/flink-filesystems/flink-azure-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-azure-fs-hadoop/pom.xml
@@ -34,7 +34,6 @@ under the License.
<!-- need to use a release which includes this patch: https://github.com/apache/hadoop/commit/02cadbd24bf69925078d044701741e2e3fcb4b2f -->
<properties>
- <fs.azure.version>2.7.0</fs.azure.version>
<fs.azure.sdk.version>1.16.0</fs.azure.sdk.version>
<fs.jackson.core.version>2.9.4</fs.jackson.core.version>
</properties>
@@ -55,16 +54,21 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-azure</artifactId>
- <version>${fs.azure.version}</version>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-fs-hadoop-shaded</artifactId>
+ <version>${project.version}</version>
</dependency>
- <!-- for the Azure HDFS related tests -->
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${fs.hadoopshaded.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- for Azure IT tests to check if HTTP endpoints are enabled / not -->
@@ -74,18 +78,6 @@ under the License.
<version>${fs.azure.sdk.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${fs.jackson.core.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- <scope>test</scope>
- </dependency>
<!-- for the behavior test suite -->
<dependency>
@@ -119,18 +111,70 @@ under the License.
</includes>
</artifactSet>
<relocations>
+ <!-- relocate the references to Hadoop to match the shaded Hadoop config -->
<relocation>
<pattern>org.apache.hadoop</pattern>
- <shadedPattern>org.apache.flink.fs.shaded.hadoop.org.apache.hadoop</shadedPattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.hadoop</shadedPattern>
</relocation>
- <!-- relocate the azure-storage dependencies -->
+
+ <!-- shade dependencies internally used by Hadoop and never exposed downstream -->
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.commons</shadedPattern>
+ </relocation>
+
+ <!-- relocate the Azure dependencies -->
<relocation>
- <pattern>com.microsoft.azure.storage</pattern>
- <shadedPattern>org.apache.flink.fs.shaded.com.microsoft.azure.storage</shadedPattern>
+ <pattern>com.microsoft.azure</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.shaded.com.microsoft.azure</shadedPattern>
+ </relocation>
+
+ <!-- shade dependencies internally used by Azure and never exposed downstream -->
+ <relocation>
+ <pattern>org.apache.httpcomponents</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.shaded.org.apache.httpcomponents</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>commons-logging</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.shaded.commons-logging</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>commons-codec</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.shaded.commons-codec</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.shaded.com.fasterxml</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.shaded.com.google</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.eclipse</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.shaded.org.eclipse</shadedPattern>
+ </relocation>
+
+ <!-- shade Flink's Hadoop FS adapter classes -->
+ <relocation>
+ <pattern>org.apache.flink.runtime.fs.hdfs</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.common.hadoop</shadedPattern>
+ </relocation>
+ <!-- shade Flink's Hadoop FS utility classes -->
+ <relocation>
+ <pattern>org.apache.flink.runtime.util</pattern>
+ <shadedPattern>org.apache.flink.fs.azure.common</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
+ <artifact>org.apache.flink:flink-hadoop-fs</artifact>
+ <excludes>
+ <exclude>org/apache/flink/runtime/util/HadoopUtils</exclude>
+ <exclude>org/apache/flink/runtime/fs/hdfs/HadoopRecoverable*</exclude>
+ </excludes>
+ </filter>
+ <filter>
<artifact>*</artifact>
<excludes>
<exclude>properties.dtd</exclude>
@@ -141,8 +185,6 @@ under the License.
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
- <exclude>core-default.xml</exclude>
- <exclude>hdfs-default.xml</exclude>
</excludes>
</filter>
</filters>