You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2017/05/15 09:31:55 UTC
[5/5] flink git commit: [FLINK-6514] [build] Create a proper separate
Hadoop uber jar for 'flink-dist' assembly
[FLINK-6514] [build] Create a proper separate Hadoop uber jar for 'flink-dist' assembly
This closes #3876
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/ae423e1d
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/ae423e1d
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/ae423e1d
Branch: refs/heads/master
Commit: ae423e1d1e591d54cd4da32ffe8835b5fea05ed0
Parents: aa13591
Author: Stephan Ewen <se...@apache.org>
Authored: Thu May 11 17:00:03 2017 +0200
Committer: Robert Metzger <rm...@apache.org>
Committed: Mon May 15 11:31:13 2017 +0200
----------------------------------------------------------------------
flink-dist/src/main/assemblies/bin.xml | 8 +-
flink-dist/src/main/assemblies/opt.xml | 7 -
.../flink-shaded-hadoop2-uber/pom.xml | 129 +++++++++++++++++++
flink-shaded-hadoop/pom.xml | 38 ++++--
4 files changed, 157 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/ae423e1d/flink-dist/src/main/assemblies/bin.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml
index cefb477..45fd792 100644
--- a/flink-dist/src/main/assemblies/bin.xml
+++ b/flink-dist/src/main/assemblies/bin.xml
@@ -33,9 +33,9 @@ under the License.
<dependencySet>
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
- <useTransitiveDependencies>true</useTransitiveDependencies>
<useProjectArtifact>false</useProjectArtifact>
<useProjectAttachments>false</useProjectAttachments>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
<useTransitiveFiltering>true</useTransitiveFiltering>
<includes>
@@ -54,11 +54,11 @@ under the License.
<fileMode>0644</fileMode>
</file>
- <!-- copy the Hadoop fat jar -->
+ <!-- copy the Hadoop uber jar -->
<file>
- <source>../flink-shaded-hadoop/flink-shaded-hadoop2/target/flink-shaded-hadoop2-${project.version}.jar</source>
+ <source>../flink-shaded-hadoop/flink-shaded-hadoop2-uber/target/flink-shaded-hadoop2-uber-${project.version}.jar</source>
<outputDirectory>lib/</outputDirectory>
- <destName>flink-shaded-hadoop2-${project.version}.jar</destName>
+ <destName>flink-shaded-hadoop2-uber-${project.version}.jar</destName>
<fileMode>0644</fileMode>
</file>
http://git-wip-us.apache.org/repos/asf/flink/blob/ae423e1d/flink-dist/src/main/assemblies/opt.xml
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 23846be..bb04d28 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -110,12 +110,5 @@
<destName>flink-metrics-datadog-${project.version}.jar</destName>
<fileMode>0644</fileMode>
</file>
-
- <file>
- <source>../flink-shaded-hadoop/flink-shaded-hadoop2/target/flink-shaded-hadoop2-${project.version}.jar</source>
- <outputDirectory>opt/</outputDirectory>
- <destName>flink-shaded-hadoop2-${project.version}.jar</destName>
- <fileMode>0644</fileMode>
- </file>
</files>
</assembly>
http://git-wip-us.apache.org/repos/asf/flink/blob/ae423e1d/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml
new file mode 100644
index 0000000..1ccff32
--- /dev/null
+++ b/flink-shaded-hadoop/flink-shaded-hadoop2-uber/pom.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-hadoop</artifactId>
+ <version>1.4-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-shaded-hadoop2-uber</artifactId>
+ <name>flink-shaded-hadoop2-uber</name>
+
+ <packaging>jar</packaging>
+
+ <!--
+ the only dependency of the 'flink-shaded-hadoop2' artifact, out
+ of which we build a fat jar
+ -->
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-hadoop2</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+
+
+ <build>
+ <plugins>
+
+ <!--
+ Don't deploy this uber-jar. It is not referenced by any other artifact.
+ Its sole purpose is to be included in the 'flink-dist' build.
+ -->
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ <!--
+ Build an uber jar of the shaded-hadoop-dependency
+ and all its transitive dependencies
+ -->
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-hadoop</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+
+ <filters>
+ <!-- Exclude signatures -->
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ <excludes>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>log4j:*</exclude>
+ </excludes>
+ </artifactSet>
+
+ <transformers>
+ <!-- The service transformer is needed to merge META-INF/services files -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+ <!-- The ApacheLicenseResourceTransformer prevents duplicate Apache Licenses -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
+ </transformers>
+
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/ae423e1d/flink-shaded-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-hadoop/pom.xml b/flink-shaded-hadoop/pom.xml
index 1d957ab..a549ae9 100644
--- a/flink-shaded-hadoop/pom.xml
+++ b/flink-shaded-hadoop/pom.xml
@@ -36,15 +36,31 @@ under the License.
<modules>
<module>flink-shaded-hadoop2</module>
+ <module>flink-shaded-hadoop2-uber</module>
</modules>
<dependencies>
- <!-- Flink already includes JSR 305. Setting this to provided excludes it from the dependencies-->
+ <!-- drop dependencies that are already provided by Flink from the shaded / uber jars -->
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<profiles>
@@ -92,24 +108,14 @@ under the License.
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
- <filter>
- <artifact>org.slf4j:*</artifact>
- <excludes>
- <exclude>org/slf4j/impl/**</exclude>
- </excludes>
- </filter>
- <!-- Exclude Hadoop's log4j. Hadoop can use Flink's log4j dependency -->
- <filter>
- <artifact>log4j:*</artifact>
- <excludes>
- <exclude>org/apache/log4j/**</exclude>
- </excludes>
- </filter>
</filters>
<transformers>
<!-- The service transformer is needed to merge META-INF/services files -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <!-- The ApacheNoticeResourceTransformer collects and aggregates NOTICE files -->
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+ <!-- The ApacheLicenseResourceTransformer prevents duplicate Apache Licenses -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
</transformers>
<artifactSet>
<includes>
@@ -127,6 +133,10 @@ under the License.
<include>org.apache.httpcomponents:*</include>
<include>commons-httpclient:commons-httpclient</include>
</includes>
+ <excludes>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>log4j:*</exclude>
+ </excludes>
</artifactSet>
<relocations>
<relocation>