You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2020/05/13 14:28:55 UTC
[flink] 01/08: [FLINK-11086] Replace flink-shaded-hadoop-2
dependency by vanilla Hadoop dependency
This is an automated email from the ASF dual-hosted git repository.
rmetzger pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit e1e7d7f7ecc080c850a264021bf1b20e3d27d373
Author: Robert Metzger <rm...@apache.org>
AuthorDate: Mon May 4 10:07:17 2020 +0200
[FLINK-11086] Replace flink-shaded-hadoop-2 dependency by vanilla Hadoop dependency
---
.../flink-connector-filesystem/pom.xml | 16 +++-
flink-connectors/flink-connector-hive/pom.xml | 95 +++++++++++++++++++++-
.../flink-hadoop-compatibility/pom.xml | 10 ++-
flink-connectors/flink-hbase/pom.xml | 4 +-
flink-connectors/flink-hcatalog/pom.xml | 10 ++-
flink-dist/pom.xml | 48 +----------
flink-dist/src/main/assemblies/hadoop.xml | 41 ----------
.../flink-bucketing-sink-test/pom.xml | 9 +-
flink-end-to-end-tests/pom.xml | 15 ++++
flink-filesystems/flink-hadoop-fs/pom.xml | 17 +++-
flink-filesystems/flink-mapr-fs/pom.xml | 10 ++-
flink-filesystems/flink-s3-fs-hadoop/pom.xml | 11 +++
flink-filesystems/flink-swift-fs-hadoop/pom.xml | 24 +++++-
flink-formats/flink-compress/pom.xml | 10 ++-
flink-formats/flink-orc-nohive/pom.xml | 29 ++++++-
flink-formats/flink-orc/pom.xml | 30 ++++---
flink-formats/flink-parquet/pom.xml | 10 ++-
flink-formats/flink-sequence-file/pom.xml | 9 +-
flink-fs-tests/pom.xml | 26 +++++-
flink-runtime/pom.xml | 16 +++-
flink-tests/pom.xml | 8 +-
flink-yarn-tests/pom.xml | 24 ------
flink-yarn/pom.xml | 25 +++++-
pom.xml | 61 ++++++++++++--
24 files changed, 390 insertions(+), 168 deletions(-)
diff --git a/flink-connectors/flink-connector-filesystem/pom.xml b/flink-connectors/flink-connector-filesystem/pom.xml
index 5a7a94e..5b6489a 100644
--- a/flink-connectors/flink-connector-filesystem/pom.xml
+++ b/flink-connectors/flink-connector-filesystem/pom.xml
@@ -51,8 +51,20 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>provided</scope>
</dependency>
diff --git a/flink-connectors/flink-connector-hive/pom.xml b/flink-connectors/flink-connector-hive/pom.xml
index 48ae9edd..9dd9426 100644
--- a/flink-connectors/flink-connector-hive/pom.xml
+++ b/flink-connectors/flink-connector-hive/pom.xml
@@ -41,6 +41,91 @@ under the License.
<derby.version>10.10.2.0</derby.version>
</properties>
+ <!-- Overwrite hadoop dependency management from flink-parent to use locally defined Hadoop version -->
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hivemetastore.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<!-- core dependencies -->
@@ -126,12 +211,16 @@ under the License.
thus override the default hadoop version from 2.4.1 to 2.7.5
-->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2-uber</artifactId>
- <version>${hivemetastore.hadoop.version}-${flink.shaded.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
<!-- Hive dependencies -->
<!-- Note: Hive published jars do not have proper dependencies declared.
We need to push for HIVE-16391 (https://issues.apache.org/jira/browse/HIVE-16391) to resolve this problem. -->
diff --git a/flink-connectors/flink-hadoop-compatibility/pom.xml b/flink-connectors/flink-hadoop-compatibility/pom.xml
index 46237dd..efcc96c 100644
--- a/flink-connectors/flink-hadoop-compatibility/pom.xml
+++ b/flink-connectors/flink-hadoop-compatibility/pom.xml
@@ -61,8 +61,14 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>provided</scope>
</dependency>
diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml
index 6c46bbd..cd31cfb 100644
--- a/flink-connectors/flink-hbase/pom.xml
+++ b/flink-connectors/flink-hbase/pom.xml
@@ -101,8 +101,8 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
diff --git a/flink-connectors/flink-hcatalog/pom.xml b/flink-connectors/flink-hcatalog/pom.xml
index 1eb6cc4..fc84696 100644
--- a/flink-connectors/flink-hcatalog/pom.xml
+++ b/flink-connectors/flink-hcatalog/pom.xml
@@ -82,8 +82,14 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>provided</scope>
</dependency>
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 3225fa0..59ada96 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -137,8 +137,8 @@ under the License.
<version>${project.version}</version>
<exclusions>
<exclusion>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -461,50 +461,6 @@ under the License.
</profile>
<profile>
- <!-- Copies that shaded Hadoop uber jar to the dist folder. -->
- <id>include-hadoop</id>
- <activation>
- <property>
- <name>include-hadoop</name>
- </property>
- </activation>
- <dependencies>
- <!--
- The Hadoop 2 Uber jar should not go into the Flink dist jar, but
- sit next to it. Hence, we set it to 'provided' here.
- -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2-uber</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>hadoop</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptors>
- <descriptor>src/main/assemblies/hadoop.xml</descriptor>
- </descriptors>
- <finalName>flink-${project.version}-bin</finalName>
- <appendAssemblyId>false</appendAssemblyId>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
- </profile>
- <profile>
<!-- Creates/Removes the 'build-target' symlink in the root directory (only Unix systems) -->
<id>symlink-build-target</id>
<activation>
diff --git a/flink-dist/src/main/assemblies/hadoop.xml b/flink-dist/src/main/assemblies/hadoop.xml
deleted file mode 100644
index 7a3ebc4..0000000
--- a/flink-dist/src/main/assemblies/hadoop.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<assembly
- xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
- <id>hadoop</id>
- <formats>
- <format>dir</format>
- </formats>
-
- <includeBaseDirectory>true</includeBaseDirectory>
- <baseDirectory>flink-${project.version}</baseDirectory>
-
- <dependencySets>
- <dependencySet>
- <outputDirectory>lib/</outputDirectory>
- <useTransitiveDependencies>true</useTransitiveDependencies>
- <scope>provided</scope>
- <includes>
- <include>org.apache.flink:flink-shaded-hadoop-2-uber</include>
- </includes>
- </dependencySet>
- </dependencySets>
-</assembly>
diff --git a/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml b/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml
index 7e010d9..433215c 100644
--- a/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml
+++ b/flink-end-to-end-tests/flink-bucketing-sink-test/pom.xml
@@ -47,8 +47,13 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
<scope>provided</scope>
<exclusions>
<!-- Needed for proper dependency convergence -->
diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
index 8689fae..3b97256 100644
--- a/flink-end-to-end-tests/pom.xml
+++ b/flink-end-to-end-tests/pom.xml
@@ -255,6 +255,21 @@ under the License.
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dependency-convergence</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
<pluginManagement>
diff --git a/flink-filesystems/flink-hadoop-fs/pom.xml b/flink-filesystems/flink-hadoop-fs/pom.xml
index f5c247e..a8f07c6 100644
--- a/flink-filesystems/flink-hadoop-fs/pom.xml
+++ b/flink-filesystems/flink-hadoop-fs/pom.xml
@@ -43,10 +43,23 @@ under the License.
<!-- pulling in Hadoop by default -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
<optional>true</optional>
</dependency>
+
<!-- for the behavior test suite -->
<dependency>
diff --git a/flink-filesystems/flink-mapr-fs/pom.xml b/flink-filesystems/flink-mapr-fs/pom.xml
index 32725fb..1d914ea 100644
--- a/flink-filesystems/flink-mapr-fs/pom.xml
+++ b/flink-filesystems/flink-mapr-fs/pom.xml
@@ -47,8 +47,14 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
<optional>true</optional>
</dependency>
diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
index 58c5b12..db569a4 100644
--- a/flink-filesystems/flink-s3-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-s3-fs-hadoop/pom.xml
@@ -32,6 +32,17 @@ under the License.
<packaging>jar</packaging>
+ <!-- Override the flink-parent dependencyManagement definition for hadoop-common to ensure
+ ${fs.hadoopshaded.version} is used for this file system -->
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${fs.hadoopshaded.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<!-- Flink's file system abstraction (compiled against, not bundled) -->
diff --git a/flink-filesystems/flink-swift-fs-hadoop/pom.xml b/flink-filesystems/flink-swift-fs-hadoop/pom.xml
index 2307a1e..b63a068 100644
--- a/flink-filesystems/flink-swift-fs-hadoop/pom.xml
+++ b/flink-filesystems/flink-swift-fs-hadoop/pom.xml
@@ -37,6 +37,26 @@ under the License.
<openstackhadoop.hadoop.version>2.8.1</openstackhadoop.hadoop.version>
</properties>
+ <!-- Overwrite hadoop dependency versions inherited from the parent pom dependencyManagement -->
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${openstackhadoop.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${openstackhadoop.hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${openstackhadoop.hadoop.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<dependencies>
<!-- Flink core -->
@@ -59,8 +79,8 @@ under the License.
because the optional Hadoop dependency is also pulled in for tests -->
<exclusions>
<exclusion>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
</exclusion>
</exclusions>
</dependency>
diff --git a/flink-formats/flink-compress/pom.xml b/flink-formats/flink-compress/pom.xml
index 6446944..84a73aa 100644
--- a/flink-formats/flink-compress/pom.xml
+++ b/flink-formats/flink-compress/pom.xml
@@ -44,8 +44,14 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
<scope>provided</scope>
</dependency>
diff --git a/flink-formats/flink-orc-nohive/pom.xml b/flink-formats/flink-orc-nohive/pom.xml
index 3ef6cec..f352313 100644
--- a/flink-formats/flink-orc-nohive/pom.xml
+++ b/flink-formats/flink-orc-nohive/pom.xml
@@ -63,7 +63,7 @@ under the License.
<version>${orc.version}</version>
<classifier>nohive</classifier>
<exclusions>
- <!-- Exclude ORC's Hadoop dependency and pull in Flink's shaded Hadoop. -->
+ <!-- Exclude ORC's Hadoop dependency and pull in Flink's Hadoop. -->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -81,12 +81,17 @@ under the License.
<!-- Replacement for ORC's Hadoop dependency. -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
- <version>${hadoop.version}-${flink.shaded.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
<!-- test dependencies -->
<dependency>
<groupId>org.apache.flink</groupId>
@@ -98,6 +103,22 @@ under the License.
</dependencies>
+ <profiles>
+ <profile>
+ <!-- This profile adds dependencies needed to execute the tests
+ with Hadoop 3 -->
+ <id>hadoop3-tests</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<build>
<plugins>
<!-- skip dependency convergence due to Hadoop dependency -->
diff --git a/flink-formats/flink-orc/pom.xml b/flink-formats/flink-orc/pom.xml
index 3aa36cb..00a39e2 100644
--- a/flink-formats/flink-orc/pom.xml
+++ b/flink-formats/flink-orc/pom.xml
@@ -75,15 +75,6 @@ under the License.
<artifactId>orc-core</artifactId>
<version>${orc.version}</version>
<exclusions>
- <!-- Exclude ORC's Hadoop dependency and pull in Flink's shaded Hadoop. -->
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </exclusion>
<exclusion>
<groupId>javax.activation</groupId>
<artifactId>javax.activation-api</artifactId>
@@ -97,8 +88,8 @@ under the License.
<!-- Replacement for ORC's Hadoop dependency. -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
@@ -147,6 +138,23 @@ under the License.
</dependencies>
+ <profiles>
+ <profile>
+ <!-- This profile adds dependencies needed to execute the tests
+ with Hadoop 3 -->
+ <id>hadoop3-tests</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
+
<build>
<plugins>
<!-- skip dependency convergence due to Hadoop dependency -->
diff --git a/flink-formats/flink-parquet/pom.xml b/flink-formats/flink-parquet/pom.xml
index 00b1d3e..1ca05fb 100644
--- a/flink-formats/flink-parquet/pom.xml
+++ b/flink-formats/flink-parquet/pom.xml
@@ -97,8 +97,14 @@ under the License.
<!-- Hadoop is needed by Parquet -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>provided</scope>
</dependency>
diff --git a/flink-formats/flink-sequence-file/pom.xml b/flink-formats/flink-sequence-file/pom.xml
index 2d0bc88..9f99598 100644
--- a/flink-formats/flink-sequence-file/pom.xml
+++ b/flink-formats/flink-sequence-file/pom.xml
@@ -45,11 +45,16 @@ under the License.
<!-- Hadoop is needed for SequenceFile -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
<!-- test dependencies -->
<dependency>
diff --git a/flink-fs-tests/pom.xml b/flink-fs-tests/pom.xml
index da60966..80230b9 100644
--- a/flink-fs-tests/pom.xml
+++ b/flink-fs-tests/pom.xml
@@ -37,8 +37,14 @@ under the License.
-->
<dependencies>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
<scope>test</scope>
</dependency>
@@ -130,6 +136,22 @@ under the License.
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <!-- This profile adds dependencies needed to execute the tests
+ with Hadoop 3 -->
+ <id>hadoop3-tests</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<build>
<plugins>
<plugin>
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index f4366a6..5c51e73 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -68,8 +68,20 @@ under the License.
<!-- optional dependency on Hadoop, so that Hadoop classes are not always pulled in -->
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
<optional>true</optional>
</dependency>
diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml
index b62c8d6..53f781c 100644
--- a/flink-tests/pom.xml
+++ b/flink-tests/pom.xml
@@ -58,12 +58,10 @@ under the License.
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
<scope>test</scope>
- </dependency>
-
+</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-jackson</artifactId>
diff --git a/flink-yarn-tests/pom.xml b/flink-yarn-tests/pom.xml
index 0e3b932..2f5e9f4 100644
--- a/flink-yarn-tests/pom.xml
+++ b/flink-yarn-tests/pom.xml
@@ -90,14 +90,6 @@ under the License.
<artifactId>flink-yarn_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <!-- prevent flink-shaded-hadoop from being on the test classpath
- to avoid conflicts with other dependencies -->
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
@@ -106,14 +98,6 @@ under the License.
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <!-- prevent flink-shaded-hadoop from being on the test classpath
- to avoid conflicts with other dependencies -->
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
@@ -401,14 +385,6 @@ under the License.
<artifactItems>
<artifactItem>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2-uber</artifactId>
- <version>${hadoop.version}-${flink.shaded.version}</version>
- <type>jar</type>
- <overWrite>true</overWrite>
- <outputDirectory>${project.build.directory}/shaded-hadoop</outputDirectory>
- </artifactItem>
- <artifactItem>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-examples-batch_${scala.binary.version}</artifactId>
<type>jar</type>
<classifier>WordCount</classifier>
diff --git a/flink-yarn/pom.xml b/flink-yarn/pom.xml
index a9bec80..ac35e6b 100644
--- a/flink-yarn/pom.xml
+++ b/flink-yarn/pom.xml
@@ -54,9 +54,30 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
</dependency>
+
<!-- test dependencies -->
diff --git a/pom.xml b/pom.xml
index 0599b4b..9d47193 100644
--- a/pom.xml
+++ b/pom.xml
@@ -308,9 +308,9 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2</artifactId>
- <version>${hadoop.version}-${flink.shaded.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
@@ -324,9 +324,57 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2-uber</artifactId>
- <version>${hadoop.version}-${flink.shaded.version}</version>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
@@ -1482,6 +1530,7 @@ under the License.
<forkNumber>0${surefire.forkNumber}</forkNumber>
<log4j.configuration>${log4j.configuration}</log4j.configuration>
<jobmanager.scheduler>${test.scheduler.type}</jobmanager.scheduler>
+ <hadoop.version>${hadoop.version}</hadoop.version>
</systemPropertyVariables>
<argLine>-Xms256m -Xmx2048m -Dmvn.forkNumber=${surefire.forkNumber} -XX:+UseG1GC</argLine>
</configuration>