You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2018/02/08 16:18:07 UTC
[2/2] flink git commit: [FLINK-8362][elasticsearch] shade all
dependencies
[FLINK-8362][elasticsearch] shade all dependencies
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c2396628
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c2396628
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c2396628
Branch: refs/heads/master
Commit: c2396628a02946c2cbf56eb3c12d1cbb90dea003
Parents: 9435370
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Thu Jan 4 21:09:09 2018 +0100
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Thu Feb 8 17:17:20 2018 +0100
----------------------------------------------------------------------
docs/dev/connectors/elasticsearch.md | 33 +----
.../flink-connector-elasticsearch/pom.xml | 100 +++++++++++++++
.../flink-connector-elasticsearch2/pom.xml | 125 ++++++++++++++++++
.../flink-connector-elasticsearch5/pom.xml | 126 +++++++++++++++++++
tools/travis_mvn_watchdog.sh | 29 +++++
5 files changed, 382 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c2396628/docs/dev/connectors/elasticsearch.md
----------------------------------------------------------------------
diff --git a/docs/dev/connectors/elasticsearch.md b/docs/dev/connectors/elasticsearch.md
index 8774fcb..52d1b58 100644
--- a/docs/dev/connectors/elasticsearch.md
+++ b/docs/dev/connectors/elasticsearch.md
@@ -440,36 +440,7 @@ For the execution of your Flink program, it is recommended to build a
so-called uber-jar (executable jar) containing all your dependencies
(see [here]({{site.baseurl}}/dev/linking.html) for further information).
-However, when an uber-jar containing an Elasticsearch sink is executed,
-an `IllegalArgumentException` may occur, which is caused by conflicting
-files of Elasticsearch and it's dependencies in `META-INF/services`:
-
-```
-IllegalArgumentException[An SPI class of type org.apache.lucene.codecs.PostingsFormat with name 'Lucene50' does not exist. You need to add the corresponding JAR file supporting this SPI to your classpath. The current classpath supports the following names: [es090, completion090, XBloomFilter]]
-```
-
-If the uber-jar is built using Maven, this issue can be avoided by
-adding the following to the Maven POM file in the plugins section:
-
-~~~xml
-<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.4.3</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- </transformers>
- </configuration>
- </execution>
- </executions>
-</plugin>
-~~~
+Alternatively, you can put the connector's jar file into Flink's `lib/` folder to make it available
+system-wide, i.e. for all job being run.
{% top %}
http://git-wip-us.apache.org/repos/asf/flink/blob/c2396628/flink-connectors/flink-connector-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-elasticsearch/pom.xml b/flink-connectors/flink-connector-elasticsearch/pom.xml
index 778a1d8..390d653 100644
--- a/flink-connectors/flink-connector-elasticsearch/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch/pom.xml
@@ -93,6 +93,106 @@ under the License.
<rerunFailingTestsCount>3</rerunFailingTestsCount>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <!-- elasticsearch 1.x already shades this but forgets the service file -->
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.elasticsearch.common.jackson</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.spatial4j</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.com.spatial4j</shadedPattern>
+ </relocation>
+
+ <!-- relocate everything from the flink-connector-elasticsearch base project -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.connectors.elasticsearch.</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.apache.flink.streaming.connectors.elasticsearch.</shadedPattern>
+ <excludes>
+ <!-- keep this project's classes as they are -->
+ <exclude>org.apache.flink.streaming.connectors.elasticsearch.Elasticsearch1ApiCallBridge</exclude>
+ <exclude>org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSink</exclude>
+ <exclude>org.apache.flink.streaming.connectors.elasticsearch.IndexRequestBuilder</exclude>
+ <exclude>org.apache.flink.streaming.connectors.elasticsearch.IndexRequestBuilderWrapperFunction</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.apache</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.apache</shadedPattern>
+ <excludes>
+ <!-- keep flink classes as they are (exceptions as above) -->
+ <exclude>org.apache.flink.**</exclude>
+ <exclude>org.apache.log4j.**</exclude> <!-- provided -->
+ </excludes>
+ </relocation>
+
+ <relocation>
+ <pattern>org.antlr</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.antlr</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.elasticsearch</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.elasticsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.joda</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.joda</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.tartarus</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.tartarus</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.yaml</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch.shaded.org.yaml</shadedPattern>
+ </relocation>
+
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>log4j.properties</exclude>
+ <exclude>config/favicon.ico</exclude>
+ <exclude>mozilla/**</exclude>
+ <exclude>META-INF/maven/com*/**</exclude>
+ <exclude>META-INF/maven/io*/**</exclude>
+ <exclude>META-INF/maven/joda*/**</exclude>
+ <exclude>META-INF/maven/net*/**</exclude>
+ <exclude>META-INF/maven/org.an*/**</exclude>
+ <exclude>META-INF/maven/org.apache.h*/**</exclude>
+ <exclude>META-INF/maven/org.apache.commons/**</exclude>
+ <exclude>META-INF/maven/org.apache.flink/flink-connector-elasticsearch-base*/**</exclude>
+ <exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
+ <exclude>META-INF/maven/org.apache.logging*/**</exclude>
+ <exclude>META-INF/maven/org.e*/**</exclude>
+ <exclude>META-INF/maven/org.h*/**</exclude>
+ <exclude>META-INF/maven/org.j*/**</exclude>
+ <exclude>META-INF/maven/org.y*/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/flink/blob/c2396628/flink-connectors/flink-connector-elasticsearch2/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-elasticsearch2/pom.xml b/flink-connectors/flink-connector-elasticsearch2/pom.xml
index 6049b3b..59ec9ee 100644
--- a/flink-connectors/flink-connector-elasticsearch2/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch2/pom.xml
@@ -91,4 +91,129 @@ under the License.
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.carrotsearch</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.carrotsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.fasterxml</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.google</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.ning</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.ning</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.spatial4j</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.spatial4j</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.tdunning</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.twitter</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.com.twitter</shadedPattern>
+ </relocation>
+
+ <!-- relocate everything from the flink-connector-elasticsearch base project -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.connectors.elasticsearch</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.apache.flink.streaming.connectors.elasticsearch</shadedPattern>
+ <excludes>
+ <!-- keep this project's classes as they are -->
+ <exclude>org.apache.flink.streaming.connectors.elasticsearch2.**</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.apache</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.apache</shadedPattern>
+ <excludes>
+ <!-- keep flink classes as they are (exceptions as above) -->
+ <exclude>org.apache.flink.**</exclude>
+ <exclude>org.apache.log4j.**</exclude> <!-- provided -->
+ </excludes>
+ </relocation>
+
+ <relocation>
+ <pattern>org.elasticsearch</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.elasticsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.jboss</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.jboss</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.joda</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.joda</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.HdrHistogram</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.HdrHistogram</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.tartarus</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.tartarus</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.yaml</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch2.shaded.org.yaml</shadedPattern>
+ </relocation>
+
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>log4j.properties</exclude>
+ <exclude>config/favicon.ico</exclude>
+ <exclude>mozilla/**</exclude>
+ <exclude>META-INF/maven/com*/**</exclude>
+ <exclude>META-INF/maven/io*/**</exclude>
+ <exclude>META-INF/maven/joda*/**</exclude>
+ <exclude>META-INF/maven/net*/**</exclude>
+ <exclude>META-INF/maven/org.an*/**</exclude>
+ <exclude>META-INF/maven/org.apache.h*/**</exclude>
+ <exclude>META-INF/maven/org.apache.commons/**</exclude>
+ <exclude>META-INF/maven/org.apache.flink/flink-connector-elasticsearch-base*/**</exclude>
+ <exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
+ <exclude>META-INF/maven/org.apache.logging*/**</exclude>
+ <exclude>META-INF/maven/org.e*/**</exclude>
+ <exclude>META-INF/maven/org.h*/**</exclude>
+ <exclude>META-INF/maven/org.j*/**</exclude>
+ <exclude>META-INF/maven/org.y*/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/c2396628/flink-connectors/flink-connector-elasticsearch5/pom.xml
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-elasticsearch5/pom.xml b/flink-connectors/flink-connector-elasticsearch5/pom.xml
index cf23160..be620ab 100644
--- a/flink-connectors/flink-connector-elasticsearch5/pom.xml
+++ b/flink-connectors/flink-connector-elasticsearch5/pom.xml
@@ -155,6 +155,132 @@ under the License.
</classpathDependencyExcludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadeTestJar>false</shadeTestJar>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.carrotsearch</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.carrotsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.fasterxml</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.github</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.github</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.sun</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.sun</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.github</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.github</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.com.tdunning</shadedPattern>
+ </relocation>
+
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.io.netty</shadedPattern>
+ </relocation>
+
+ <relocation>
+ <pattern>joptsimple</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.joptsimple</shadedPattern>
+ </relocation>
+
+ <!-- relocate everything from the flink-connector-elasticsearch base project -->
+ <relocation>
+ <pattern>org.apache.flink.streaming.connectors.elasticsearch</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.apache.flink.streaming.connectors.elasticsearch</shadedPattern>
+ <excludes>
+ <!-- keep this project's classes as they are -->
+ <exclude>org.apache.flink.streaming.connectors.elasticsearch5.**</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>org.apache</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.apache</shadedPattern>
+ <excludes>
+ <!-- keep flink classes as they are (exceptions as above) -->
+ <exclude>org.apache.flink.**</exclude>
+ <exclude>org.apache.log4j.**</exclude> <!-- provided -->
+ </excludes>
+ </relocation>
+
+ <relocation>
+ <pattern>org.elasticsearch</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.elasticsearch</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.HdrHistogram</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.HdrHistogram</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.jboss</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.jboss</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.joda</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.joda</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.tartarus</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.tartarus</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.yaml</pattern>
+ <shadedPattern>org.apache.flink.streaming.connectors.elasticsearch5.shaded.org.yaml</shadedPattern>
+ </relocation>
+
+ </relocations>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>log4j.properties</exclude>
+ <exclude>config/favicon.ico</exclude>
+ <exclude>mozilla/**</exclude>
+ <exclude>META-INF/maven/com*/**</exclude>
+ <exclude>META-INF/maven/io*/**</exclude>
+ <exclude>META-INF/maven/joda*/**</exclude>
+ <exclude>META-INF/maven/net*/**</exclude>
+ <exclude>META-INF/maven/org.an*/**</exclude>
+ <exclude>META-INF/maven/org.apache.h*/**</exclude>
+ <exclude>META-INF/maven/org.apache.commons/**</exclude>
+ <exclude>META-INF/maven/org.apache.flink/flink-connector-elasticsearch-base*/**</exclude>
+ <exclude>META-INF/maven/org.apache.flink/force-shading/**</exclude>
+ <exclude>META-INF/maven/org.apache.logging*/**</exclude>
+ <exclude>META-INF/maven/org.e*/**</exclude>
+ <exclude>META-INF/maven/org.h*/**</exclude>
+ <exclude>META-INF/maven/org.j*/**</exclude>
+ <exclude>META-INF/maven/org.y*/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/flink/blob/c2396628/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 884e275..55dd465 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -444,6 +444,32 @@ check_shaded_artifacts_s3_fs() {
return 0
}
+# Check the elasticsearch connectors' fat jars for illegal or missing artifacts
+check_shaded_artifacts_connector_elasticsearch() {
+ VARIANT=$1
+ find flink-connectors/flink-connector-elasticsearch${VARIANT}/target/flink-connector-elasticsearch${VARIANT}*.jar ! -name "*-tests.jar" -exec jar tf {} \; > allClasses
+
+ UNSHADED_CLASSES=`cat allClasses | grep -v -e '^META-INF' -e '^assets' -e "^org/apache/flink/streaming/connectors/elasticsearch${VARIANT}/" | grep '\.class$'`
+ if [ "$?" == "0" ]; then
+ echo "=============================================================================="
+ echo "Detected unshaded dependencies in flink-connector-elasticsearch${VARIANT}'s fat jar:"
+ echo "${UNSHADED_CLASSES}"
+ echo "=============================================================================="
+ return 1
+ fi
+
+ UNSHADED_SERVICES=`cat allClasses | grep '^META-INF/services/' | grep -v -e '^META-INF/services/org\.apache\.flink\.core\.fs\.FileSystemFactory$' -e "^META-INF/services/org\.apache\.flink\.fs\.s3${VARIANT}\.shaded" -e '^META-INF/services/'`
+ if [ "$?" == "0" ]; then
+ echo "=============================================================================="
+ echo "Detected unshaded service files in flink-connector-elasticsearch${VARIANT}'s fat jar:"
+ echo "${UNSHADED_SERVICES}"
+ echo "=============================================================================="
+ return 1
+ fi
+
+ return 0
+}
+
# =============================================================================
# WATCHDOG
# =============================================================================
@@ -523,6 +549,9 @@ case $TEST in
check_shaded_artifacts_s3_fs hadoop
EXIT_CODE=$(($EXIT_CODE+$?))
check_shaded_artifacts_s3_fs presto
+ check_shaded_artifacts_connector_elasticsearch ""
+ check_shaded_artifacts_connector_elasticsearch 2
+ check_shaded_artifacts_connector_elasticsearch 5
EXIT_CODE=$(($EXIT_CODE+$?))
else
echo "=============================================================================="