You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/08/31 16:14:21 UTC
flink git commit: [FLINK-2584] Check for unshaded classes in fat jar
and shade curator
Repository: flink
Updated Branches:
refs/heads/release-0.10.0-milestone-1 13a95961f -> 1133aa8b1
[FLINK-2584] Check for unshaded classes in fat jar and shade curator
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1133aa8b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1133aa8b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1133aa8b
Branch: refs/heads/release-0.10.0-milestone-1
Commit: 1133aa8b1f19194ee563b32acdc836e3abab44d7
Parents: 13a9596
Author: Robert Metzger <rm...@apache.org>
Authored: Thu Aug 27 18:13:08 2015 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Mon Aug 31 13:37:16 2015 +0200
----------------------------------------------------------------------
.../flink-storm-compatibility-core/pom.xml | 5 ++
flink-runtime/pom.xml | 5 +-
flink-shaded-curator/pom.xml | 89 ++++++++++++++++++++
.../flink-connector-elasticsearch/pom.xml | 6 ++
.../flink-connector-kafka/pom.xml | 6 ++
pom.xml | 7 +-
tools/travis_mvn_watchdog.sh | 23 +++++
7 files changed, 133 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1133aa8b/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml b/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml
index 92be9db..cced678 100644
--- a/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml
+++ b/flink-contrib/flink-storm-compatibility/flink-storm-compatibility-core/pom.xml
@@ -57,6 +57,11 @@ under the License.
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/flink/blob/1133aa8b/flink-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml
index 6dc8d42..ff83168 100644
--- a/flink-runtime/pom.xml
+++ b/flink-runtime/pom.xml
@@ -193,8 +193,9 @@ under the License.
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-curator</artifactId>
+ <version>${project.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/1133aa8b/flink-shaded-curator/pom.xml
----------------------------------------------------------------------
diff --git a/flink-shaded-curator/pom.xml b/flink-shaded-curator/pom.xml
new file mode 100644
index 0000000..e1c938c
--- /dev/null
+++ b/flink-shaded-curator/pom.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-parent</artifactId>
+ <version>0.10-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+
+ <artifactId>flink-shaded-curator</artifactId>
+ <name>flink-shaded-curator</name>
+
+ <packaging>jar</packaging>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curatorrecipes.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!-- Relocate the Curator's Guava dependency into a different namespace and
+ put create our own apache curator dependency.
+ We can easily integrate curator's netty into the jar file.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id> <!-- override inherited execution id -->
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <dependencyReducedPomLocation>${project.basedir}/target/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+ <transformers>
+ <!-- The service transformer is needed to merge META-INF/services files -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/>
+ </transformers>
+ <artifactSet>
+ <includes>
+ <include>org.apache.curator:*</include>
+ <include>com.google.guava:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>org.apache.curator.shaded.com.google</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/1133aa8b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
index ccce746..d4a009f 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-elasticsearch/pom.xml
@@ -55,6 +55,12 @@ under the License.
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-core</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/1133aa8b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
index bbbd2be..98d7f0d 100644
--- a/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
+++ b/flink-staging/flink-streaming/flink-streaming-connectors/flink-connector-kafka/pom.xml
@@ -93,6 +93,12 @@ under the License.
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<version>2.7.1</version>
http://git-wip-us.apache.org/repos/asf/flink/blob/1133aa8b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dbba9fc..63ad247 100644
--- a/pom.xml
+++ b/pom.xml
@@ -51,6 +51,7 @@ under the License.
<modules>
<module>flink-shaded-hadoop</module>
+ <module>flink-shaded-curator</module>
<module>flink-core</module>
<module>flink-java</module>
<module>flink-scala</module>
@@ -336,12 +337,6 @@ under the License.
<artifactId>zookeeper</artifactId>
<version>${zookeeper.version}</version>
</dependency>
-
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curatorrecipes.version}</version>
- </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/flink/blob/1133aa8b/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 08cf6ef..629c44c 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -156,6 +156,27 @@ watchdog () {
done
}
+# Check the final fat jar for illegal artifacts
+check_shaded_artifacts() {
+ jar tf build-target/lib/flink-dist-*.jar > allClasses
+ ASM=`cat allClasses | grep '^org/objectweb/asm/' | wc -l`
+ if [ $ASM != "0" ]; then
+ echo "=============================================================================="
+ echo "Detected $ASM asm dependencies in fat jar"
+ echo "=============================================================================="
+ exit 1
+ fi
+
+ GUAVA=`cat allClasses | grep '^com/google/common' | wc -l`
+ if [ $GUAVA != "0" ]; then
+ echo "=============================================================================="
+ echo "Detected $GUAVA guava dependencies in fat jar"
+ echo "=============================================================================="
+ exit 1
+ fi
+
+}
+
# =============================================================================
# WATCHDOG
# =============================================================================
@@ -189,6 +210,8 @@ echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
rm $MVN_PID
rm $MVN_EXIT
+check_shaded_artifacts
+
put_yarn_logs_to_artifacts
upload_artifacts_s3