You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/09/19 07:16:10 UTC
[flink] 02/06: [FLINK-10366] [s3] Create an S3 base module as the
common denominator of the S3 connectors
This is an automated email from the ASF dual-hosted git repository.
sewen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 43dfa187f85836a778b031955bd026ebf3f17599
Author: Stephan Ewen <se...@apache.org>
AuthorDate: Thu Sep 13 10:48:31 2018 +0200
[FLINK-10366] [s3] Create an S3 base module as the common denominator of the S3 connectors
---
flink-filesystems/flink-s3-fs-base/pom.xml | 183 +++++++++++++++++++++++++++++
flink-filesystems/pom.xml | 1 +
2 files changed, 184 insertions(+)
diff --git a/flink-filesystems/flink-s3-fs-base/pom.xml b/flink-filesystems/flink-s3-fs-base/pom.xml
new file mode 100644
index 0000000..c1e30ac
--- /dev/null
+++ b/flink-filesystems/flink-s3-fs-base/pom.xml
@@ -0,0 +1,183 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-filesystems</artifactId>
+ <version>1.7-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-s3-fs-base</artifactId>
+ <name>flink-s3-fs-base</name>
+
+ <packaging>jar</packaging>
+
+ <properties>
+ <fs.s3.aws.version>1.11.271</fs.s3.aws.version>
+ </properties>
+
+ <dependencies>
+
+ <!-- Flink's file system abstraction (only compiled against, not bundled) -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Hadoop's file system abstraction (bundled) -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-fs-hadoop-shaded</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- The Hadoop file system adapter classes (bundled) -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-hadoop-fs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- AWS dependencies (bundled) -->
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <version>${fs.s3.aws.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${fs.s3.aws.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-kms</artifactId>
+ <version>${fs.s3.aws.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-dynamodb</artifactId>
+ <version>${fs.s3.aws.version}</version>
+ </dependency>
+
+ <!-- Hadoop's s3 support classes (bundled) -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${fs.hadoopshaded.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+
+ <!-- this is merely an intermediate build artifact and should not be -->
+ <!-- deployed to maven central -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!-- Relocate all S3 related classes -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <!-- shade dependencies internally used by Hadoop and never exposed downstream -->
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>org.apache.flink.fs.shaded.hadoop3.org.apache.commons</shadedPattern>
+ </relocation>
+
+ <!-- shade dependencies internally used by AWS and never exposed downstream -->
+ <relocation>
+ <pattern>software.amazon</pattern>
+ <shadedPattern>org.apache.flink.fs.s3base.shaded.software.amazon</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.joda</pattern>
+ <shadedPattern>org.apache.flink.fs.s3base.shaded.org.joda</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.http</pattern>
+ <shadedPattern>org.apache.flink.fs.s3base.shaded.org.apache.http</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>org.apache.flink.fs.s3base.shaded.com.fasterxml</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>org.apache.flink.fs.s3base.shaded.com.google</shadedPattern>
+ </relocation>
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>org.apache.flink:flink-hadoop-fs</artifact>
+ <excludes>
+ <exclude>org/apache/flink/runtime/util/**</exclude>
+ <exclude>org/apache/flink/runtime/fs/hdfs/HadoopRecoverable*</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>.gitkeep</exclude>
+ <exclude>mime.types</exclude>
+ <exclude>mozilla/**</exclude>
+ <exclude>META-INF/maven/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink-filesystems/pom.xml b/flink-filesystems/pom.xml
index da5c187..04018b1 100644
--- a/flink-filesystems/pom.xml
+++ b/flink-filesystems/pom.xml
@@ -42,6 +42,7 @@ under the License.
<module>flink-hadoop-fs</module>
<module>flink-mapr-fs</module>
<module>flink-fs-hadoop-shaded</module>
+ <module>flink-s3-fs-base</module>
<module>flink-s3-fs-hadoop</module>
<module>flink-s3-fs-presto</module>
<module>flink-swift-fs-hadoop</module>