You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2015/03/17 21:03:44 UTC
[1/4] flink git commit: [FLINK-1714] Fix logger of the class
SecurityUtils
Repository: flink
Updated Branches:
refs/heads/master b093469ef -> 72ca014ee
[FLINK-1714] Fix logger of the class SecurityUtils
This closes #490
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/05e06995
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/05e06995
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/05e06995
Branch: refs/heads/master
Commit: 05e06995d7099cfb33bab71aea798dcfdcddc599
Parents: b093469
Author: Hong Sibao <ho...@huawei.com>
Authored: Tue Mar 17 18:46:22 2015 +0800
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue Mar 17 20:18:34 2015 +0100
----------------------------------------------------------------------
.../java/org/apache/flink/runtime/security/SecurityUtils.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/05e06995/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
index e055b1e..5fde51e 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/security/SecurityUtils.java
@@ -17,7 +17,6 @@
*/
package org.apache.flink.runtime.security;
-import org.apache.flink.runtime.operators.MapPartitionDriver;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
@@ -27,7 +26,7 @@ import java.security.PrivilegedExceptionAction;
public class SecurityUtils {
- private static final Logger LOG = LoggerFactory.getLogger(MapPartitionDriver.class);
+ private static final Logger LOG = LoggerFactory.getLogger(SecurityUtils.class);
// load Hadoop configuration when loading the security utils.
private static Configuration hdConf = new Configuration();
[4/4] flink git commit: [docs] Remove -j and -a parameters which
seemed no longer valid in the doc example for submit job to Flink run in
YARN.
Posted by se...@apache.org.
[docs] Remove -j and -a parameters which seemed no longer valid in the doc example for submit job to Flink run in YARN.
From:
./bin/flink run -j ./examples/flink-java-examples-{{site.FLINK_VERSION_SHORT }}-WordCount.jar \
-a 1 hdfs:///..../apache-license-v2.txt hdfs:///.../wordcount-result.txt
To:
./bin/flink run ./examples/flink-java-examples-{{site.FLINK_VERSION_SHORT }}-WordCount.jar \
hdfs:///..../apache-license-v2.txt hdfs:///.../wordcount-result.txt
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/72ca014e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/72ca014e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/72ca014e
Branch: refs/heads/master
Commit: 72ca014ee1ffc59b328846967fe976ab55e6b450
Parents: 04a7388
Author: Henry Saputra <he...@gmail.com>
Authored: Thu Mar 12 16:18:03 2015 -0700
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue Mar 17 20:22:54 2015 +0100
----------------------------------------------------------------------
docs/yarn_setup.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/72ca014e/docs/yarn_setup.md
----------------------------------------------------------------------
diff --git a/docs/yarn_setup.md b/docs/yarn_setup.md
index 5c53f8b..d65cf7e 100644
--- a/docs/yarn_setup.md
+++ b/docs/yarn_setup.md
@@ -172,8 +172,8 @@ Use the *run* action to submit a job to YARN. The client is able to determine th
~~~bash
wget -O apache-license-v2.txt http://www.apache.org/licenses/LICENSE-2.0.txt
hadoop fs -copyFromLocal LICENSE-2.0.txt hdfs:/// ...
-./bin/flink run -j ./examples/flink-java-examples-{{site.FLINK_VERSION_SHORT }}-WordCount.jar \
- -a 1 hdfs:///..../apache-license-v2.txt hdfs:///.../wordcount-result.txt
+./bin/flink run ./examples/flink-java-examples-{{site.FLINK_VERSION_SHORT }}-WordCount.jar \
+ hdfs:///..../apache-license-v2.txt hdfs:///.../wordcount-result.txt
~~~
If there is the following error, make sure that all TaskManagers started:
[2/4] flink git commit: [FLINK-1414] Move quickstarts to website
Posted by se...@apache.org.
[FLINK-1414] Move quickstarts to website
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/9b4f483f
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/9b4f483f
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/9b4f483f
Branch: refs/heads/master
Commit: 9b4f483ff456dfa42a76978dbb3adb095a3f499b
Parents: 05e0699
Author: Robert Metzger <rm...@apache.org>
Authored: Tue Feb 24 10:45:37 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue Mar 17 20:21:39 2015 +0100
----------------------------------------------------------------------
docs/example_connectors.md | 2 +-
docs/java_api_quickstart.md | 2 +-
docs/scala_api_quickstart.md | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/9b4f483f/docs/example_connectors.md
----------------------------------------------------------------------
diff --git a/docs/example_connectors.md b/docs/example_connectors.md
index dccddd0..5a2e504 100644
--- a/docs/example_connectors.md
+++ b/docs/example_connectors.md
@@ -115,7 +115,7 @@ Execute the following commands:
2. Setup a new Flink project using the quickstarts:
~~~bash
- curl https://raw.githubusercontent.com/apache/flink/master/flink-quickstart/quickstart.sh | bash
+ curl http://flink.apache.org/q/quickstart.sh | bash
~~~
3. Add the following dependencies (in the `<dependencies>` section) to your `pom.xml` file:
http://git-wip-us.apache.org/repos/asf/flink/blob/9b4f483f/docs/java_api_quickstart.md
----------------------------------------------------------------------
diff --git a/docs/java_api_quickstart.md b/docs/java_api_quickstart.md
index de01b3f..89b4506 100644
--- a/docs/java_api_quickstart.md
+++ b/docs/java_api_quickstart.md
@@ -41,7 +41,7 @@ Use one of the following commands to __create a project__:
<div class="tab-content">
<div class="tab-pane active" id="quickstart-script">
{% highlight bash %}
- $ curl https://raw.githubusercontent.com/apache/flink/master/flink-quickstart/quickstart.sh | bash
+ $ curl http://flink.apache.org/q/quickstart.sh | bash
{% endhighlight %}
</div>
<div class="tab-pane" id="maven-archetype">
http://git-wip-us.apache.org/repos/asf/flink/blob/9b4f483f/docs/scala_api_quickstart.md
----------------------------------------------------------------------
diff --git a/docs/scala_api_quickstart.md b/docs/scala_api_quickstart.md
index 2472aa9..e7f841f 100644
--- a/docs/scala_api_quickstart.md
+++ b/docs/scala_api_quickstart.md
@@ -41,7 +41,7 @@ Use one of the following commands to __create a project__:
<div class="tab-content">
<div class="tab-pane active" id="quickstart-script">
{% highlight bash %}
-$ curl https://raw.githubusercontent.com/apache/flink/master/flink-quickstart/quickstart-scala.sh | bash
+$ curl http://flink.apache.org/q/quickstart-scala.sh | bash
{% endhighlight %}
</div>
<div class="tab-pane" id="maven-archetype">
[3/4] flink git commit: [FLINK-1342] Use maven-shade-plugin in
quickstarts to build fat-jar
Posted by se...@apache.org.
[FLINK-1342] Use maven-shade-plugin in quickstarts to build fat-jar
This closes #486
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/04a73883
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/04a73883
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/04a73883
Branch: refs/heads/master
Commit: 04a738836f30d8967c427208ab04b1aef34f370a
Parents: 9b4f483
Author: Robert Metzger <rm...@apache.org>
Authored: Tue Feb 24 14:02:18 2015 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Tue Mar 17 20:21:39 2015 +0100
----------------------------------------------------------------------
docs/java_api_quickstart.md | 2 +-
docs/scala_api_quickstart.md | 2 +-
.../main/resources/archetype-resources/pom.xml | 178 +++++++++++++++++--
.../src/assembly/flink-fat-jar.xml | 39 ----
.../main/resources/archetype-resources/pom.xml | 176 ++++++++++++++++--
pom.xml | 1 +
6 files changed, 322 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/04a73883/docs/java_api_quickstart.md
----------------------------------------------------------------------
diff --git a/docs/java_api_quickstart.md b/docs/java_api_quickstart.md
index 89b4506..6c0f958 100644
--- a/docs/java_api_quickstart.md
+++ b/docs/java_api_quickstart.md
@@ -68,7 +68,7 @@ A note to Mac OS X users: The default JVM heapsize for Java is too small for Fli
## Build Project
-If you want to __build your project__, go to your project directory and issue the`mvn clean package` command. You will __find a jar__ that runs on every Flink cluster in __target/your-artifact-id-1.0-SNAPSHOT.jar__. There is also a fat-jar, __target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar__. This
+If you want to __build your project__, go to your project directory and issue the`mvn clean install -Pbuild-jar` command. You will __find a jar__ that runs on every Flink cluster in __target/your-artifact-id-1.0-SNAPSHOT.jar__. There is also a fat-jar, __target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar__. This
also contains all dependencies that get added to the maven project.
## Next Steps
http://git-wip-us.apache.org/repos/asf/flink/blob/04a73883/docs/scala_api_quickstart.md
----------------------------------------------------------------------
diff --git a/docs/scala_api_quickstart.md b/docs/scala_api_quickstart.md
index e7f841f..48767cc 100644
--- a/docs/scala_api_quickstart.md
+++ b/docs/scala_api_quickstart.md
@@ -78,7 +78,7 @@ The IntelliJ IDE also supports Maven and offers a plugin for Scala development.
## Build Project
-If you want to __build your project__, go to your project directory and issue the`mvn clean package` command. You will __find a jar__ that runs on every Flink cluster in __target/your-artifact-id-1.0-SNAPSHOT.jar__. There is also a fat-jar, __target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar__. This
+If you want to __build your project__, go to your project directory and issue the `mvn clean package -Pbuild-jar` command. You will __find a jar__ that runs on every Flink cluster in __target/your-artifact-id-1.0-SNAPSHOT.jar__. There is also a fat-jar, __target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar__. This
also contains all dependencies that get added to the maven project.
## Next Steps
http://git-wip-us.apache.org/repos/asf/flink/blob/04a73883/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index d84e922..a59bde4 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -25,7 +25,7 @@ under the License.
<version>${version}</version>
<packaging>jar</packaging>
- <name>Your Job's Name</name>
+ <name>Flink Quickstart Job</name>
<url>http://www.myorganization.org</url>
<properties>
@@ -46,8 +46,24 @@ under the License.
</repository>
</repositories>
- <!-- These two requirements are the minimum to use and develop Flink.
- You can add others like <artifactId>flink-scala</artifactId> for Scala! -->
+ <!--
+
+ Execute
+ mvn clean install -Pbuild-jar
+ to build a jar file out of this project!
+
+
+ How to use the Flink Quickstart pom:
+ a) Adding new dependencies:
+ You can add dependencies to the list below.
+ Please check if the maven-shade-plugin below is filtering out your dependency and remove the exclude from there.
+
+ b) Build a jar for running on the cluster:
+ There are two options for creating a jar from this project
+ b.1) "mvn clean install" -> this will create a fat jar which contains all dependencies necessary for running the jar created by this pom in a cluster. The "maven-shade-plugin" excludes everything that is provided on a running Flink cluster.
+ b.2) "mvn clean install -Pbuild-jar" -> This will also create a fat-jar, but with much nicer dependency exclusion handling. This approach is preferred and leads to much cleaner jar files.
+
+ -->
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -68,30 +84,123 @@ under the License.
<build>
<plugins>
-
- <!-- We use the maven-assembly plugin to create a fat jar that contains all dependencies
+ <!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
except flink and it's transitive dependencies. The resulting fat-jar can be executed
on a cluster. Change the value of Program-Class if your program entry point changes. -->
<plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.4.1</version>
- <configuration>
- <descriptors>
- <descriptor>src/assembly/flink-fat-jar.xml</descriptor>
- </descriptors>
- <archive>
- <manifest>
- <mainClass>${package}.Job</mainClass>
- </manifest>
- </archive>
- </configuration>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
<executions>
+ <!-- Run shade goal on package phase -->
<execution>
- <id>make-assembly</id>
<phase>package</phase>
<goals>
- <goal>single</goal>
+ <goal>shade</goal>
</goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <!-- This list contains all dependencies of flink-dist
+ Everything else will be packaged into the fat-jar
+ -->
+ <exclude>org.apache.flink:flink-shaded-*</exclude>
+ <exclude>org.apache.flink:flink-core</exclude>
+ <exclude>org.apache.flink:flink-java</exclude>
+ <exclude>org.apache.flink:flink-scala</exclude>
+ <exclude>org.apache.flink:flink-runtime</exclude>
+ <exclude>org.apache.flink:flink-compiler</exclude>
+ <exclude>org.apache.flink:flink-spargel</exclude>
+ <exclude>org.apache.flink:flink-avro</exclude>
+ <exclude>org.apache.flink:flink-java-examples</exclude>
+ <exclude>org.apache.flink:flink-scala-examples</exclude>
+ <exclude>org.apache.flink:flink-streaming-examples</exclude>
+ <exclude>org.apache.flink:flink-streaming-core</exclude>
+ <!-- Also exclude very big transitive dependencies of Flink
+
+ WARNING: You have to remove these excludes if your code relies on other
+ versions of these dependencies.
+
+ -->
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>com.amazonaws:aws-java-sdk</exclude>
+ <exclude>io.netty:netty-all</exclude>
+ <exclude>com.typesafe.akka:akka-actor_2.10</exclude>
+ <exclude>com.typesafe.akka:akka-remote_2.10</exclude>
+ <exclude>io.netty:netty</exclude>
+ <exclude>com.typesafe.akka:akka-slf4j_2.10</exclude>
+ <exclude>org.eclipse.jetty:jetty-server</exclude>
+ <exclude>org.eclipse.jetty:jetty-continuation</exclude>
+ <exclude>org.eclipse.jetty:jetty-http</exclude>
+ <exclude>org.eclipse.jetty:jetty-io</exclude>
+ <exclude>org.eclipse.jetty:jetty-util</exclude>
+ <exclude>org.eclipse.jetty:jetty-security</exclude>
+ <exclude>org.eclipse.jetty:jetty-servlet</exclude>
+ <exclude>commons-fileupload:commons-fileupload</exclude>
+ <exclude>org.apache.avro:avro</exclude>
+ <exclude>commons-collections:commons-collections</exclude>
+ <exclude>org.codehaus.jackson:jackson-core-asl</exclude>
+ <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
+ <exclude>com.thoughtworks.paranamer:paranamer</exclude>
+ <exclude>org.xerial.snappy:snappy-java</exclude>
+ <exclude>org.apache.commons:commons-compress</exclude>
+ <exclude>org.tukaani:xz</exclude>
+ <exclude>com.esotericsoftware.kryo:kryo</exclude>
+ <exclude>com.esotericsoftware.minlog:minlog</exclude>
+ <exclude>org.objenesis:objenesis</exclude>
+ <exclude>com.twitter:chill_2.10</exclude>
+ <exclude>com.twitter:chill-java</exclude>
+ <exclude>com.twitter:chill-avro_2.10</exclude>
+ <exclude>com.twitter:chill-bijection_2.10</exclude>
+ <exclude>com.twitter:bijection-core_2.10</exclude>
+ <exclude>com.twitter:bijection-avro_2.10</exclude>
+ <exclude>com.twitter:chill-protobuf</exclude>
+ <exclude>com.google.protobuf:protobuf-java</exclude>
+ <exclude>com.twitter:chill-thrift</exclude>
+ <exclude>org.apache.thrift:libthrift</exclude>
+ <exclude>commons-lang:commons-lang</exclude>
+ <exclude>junit:junit</exclude>
+ <exclude>de.javakaffee:kryo-serializers</exclude>
+ <exclude>joda-time:joda-time</exclude>
+ <exclude>org.apache.commons:commons-lang3</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>org.apache.commons:commons-math</exclude>
+ <exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>org.apache.httpcomponents:httpclient</exclude>
+ <exclude>org.apache.httpcomponents:httpcore</exclude>
+ <exclude>commons-codec:commons-codec</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-core</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
+ <exclude>org.codehaus.jettison:jettison</exclude>
+ <exclude>stax:stax-api</exclude>
+ <exclude>com.typesafe:config</exclude>
+ <exclude>org.uncommons.maths:uncommons-maths</exclude>
+ <exclude>com.github.scopt:scopt_2.10</exclude>
+ <exclude>org.mortbay.jetty:servlet-api</exclude>
+ <exclude>commons-io:commons-io</exclude>
+ <exclude>commons-cli:commons-cli</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.flink:*</artifact>
+ <excludes>
+ <exclude>org/apache/flink/shaded/**</exclude>
+ <exclude>web-docs/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <!-- add Main-Class to manifest file -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>${package}.Job</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
</execution>
</executions>
</plugin>
@@ -100,6 +209,7 @@ under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
+ <version>2.5</version>
<configuration>
<archive>
<manifestEntries>
@@ -183,4 +293,34 @@ under the License.
-->
</build>
+ <profiles>
+ <profile>
+ <!-- A profile that does everyting correctly:
+ We set the Flink dependencies to provided -->
+ <id>build-jar</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/04a73883/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml
deleted file mode 100644
index 2819b81..0000000
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml
+++ /dev/null
@@ -1,39 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
- <id>flink-fat-jar</id>
- <formats>
- <format>jar</format>
- </formats>
- <includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>/</outputDirectory>
- <useProjectArtifact>true</useProjectArtifact>
- <excludes>
- <exclude>org.apache.flink:*</exclude>
- </excludes>
- <useTransitiveFiltering>true</useTransitiveFiltering>
- <unpack>true</unpack>
- <scope>runtime</scope>
- </dependencySet>
- </dependencySets>
-</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/04a73883/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 49036e8..43eec6d 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -26,7 +26,7 @@ under the License.
<version>${version}</version>
<packaging>jar</packaging>
- <name>Your Job's Name</name>
+ <name>Flink Quickstart Job</name>
<url>http://www.myorganization.org</url>
<repositories>
@@ -47,7 +47,24 @@ under the License.
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
- <!-- These two requirements are the minimum to use and develop Flink. -->
+ <!--
+
+ Execute
+ mvn clean install -Pbuild-jar
+ to build a jar file out of this project!
+
+
+ How to use the Flink Quickstart pom:
+ a) Adding new dependencies:
+ You can add dependencies to the list below.
+ Please check if the maven-shade-plugin below is filtering out your dependency and remove the exclude from there.
+
+ b) Build a jar for running on the cluster:
+ There are two options for creating a jar from this project
+ b.1) "mvn clean install" -> this will create a fat jar which contains all dependencies necessary for running the jar created by this pom in a cluster. The "maven-shade-plugin" excludes everything that is provided on a running Flink cluster.
+ b.2) "mvn clean install -Pbuild-jar" -> This will also create a fat-jar, but with much nicer dependency exclusion handling. This approach is preferred and leads to much cleaner jar files.
+
+ -->
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -71,26 +88,123 @@ under the License.
on a cluster. Change the value of Program-Class if your program entry point changes. -->
<build>
<plugins>
+ <!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
+ except flink and it's transitive dependencies. The resulting fat-jar can be executed
+ on a cluster. Change the value of Program-Class if your program entry point changes. -->
<plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.4.1</version>
- <configuration>
- <descriptors>
- <descriptor>src/assembly/flink-fat-jar.xml</descriptor>
- </descriptors>
- <archive>
- <manifestEntries>
- <Program-Class>${package}.Job</Program-Class>
- </manifestEntries>
- </archive>
- </configuration>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
<executions>
+ <!-- Run shade goal on package phase -->
<execution>
- <id>make-assembly</id>
<phase>package</phase>
<goals>
- <goal>single</goal>
+ <goal>shade</goal>
</goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <!-- This list contains all dependencies of flink-dist
+ Everything else will be packaged into the fat-jar
+ -->
+ <exclude>org.apache.flink:flink-shaded-*</exclude>
+ <exclude>org.apache.flink:flink-core</exclude>
+ <exclude>org.apache.flink:flink-java</exclude>
+ <exclude>org.apache.flink:flink-scala</exclude>
+ <exclude>org.apache.flink:flink-runtime</exclude>
+ <exclude>org.apache.flink:flink-compiler</exclude>
+ <exclude>org.apache.flink:flink-spargel</exclude>
+ <exclude>org.apache.flink:flink-avro</exclude>
+ <exclude>org.apache.flink:flink-java-examples</exclude>
+ <exclude>org.apache.flink:flink-scala-examples</exclude>
+ <exclude>org.apache.flink:flink-streaming-examples</exclude>
+ <exclude>org.apache.flink:flink-streaming-core</exclude>
+ <!-- Also exclude very big transitive dependencies of Flink
+
+ WARNING: You have to remove these excludes if your code relies on other
+ versions of these dependencies.
+
+ -->
+ <exclude>org.scala-lang:scala-library</exclude>
+ <exclude>com.amazonaws:aws-java-sdk</exclude>
+ <exclude>io.netty:netty-all</exclude>
+ <exclude>com.typesafe.akka:akka-actor_2.10</exclude>
+ <exclude>com.typesafe.akka:akka-remote_2.10</exclude>
+ <exclude>io.netty:netty</exclude>
+ <exclude>com.typesafe.akka:akka-slf4j_2.10</exclude>
+ <exclude>org.eclipse.jetty:jetty-server</exclude>
+ <exclude>org.eclipse.jetty:jetty-continuation</exclude>
+ <exclude>org.eclipse.jetty:jetty-http</exclude>
+ <exclude>org.eclipse.jetty:jetty-io</exclude>
+ <exclude>org.eclipse.jetty:jetty-util</exclude>
+ <exclude>org.eclipse.jetty:jetty-security</exclude>
+ <exclude>org.eclipse.jetty:jetty-servlet</exclude>
+ <exclude>commons-fileupload:commons-fileupload</exclude>
+ <exclude>org.apache.avro:avro</exclude>
+ <exclude>commons-collections:commons-collections</exclude>
+ <exclude>org.codehaus.jackson:jackson-core-asl</exclude>
+ <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
+ <exclude>com.thoughtworks.paranamer:paranamer</exclude>
+ <exclude>org.xerial.snappy:snappy-java</exclude>
+ <exclude>org.apache.commons:commons-compress</exclude>
+ <exclude>org.tukaani:xz</exclude>
+ <exclude>com.esotericsoftware.kryo:kryo</exclude>
+ <exclude>com.esotericsoftware.minlog:minlog</exclude>
+ <exclude>org.objenesis:objenesis</exclude>
+ <exclude>com.twitter:chill_2.10</exclude>
+ <exclude>com.twitter:chill-java</exclude>
+ <exclude>com.twitter:chill-avro_2.10</exclude>
+ <exclude>com.twitter:chill-bijection_2.10</exclude>
+ <exclude>com.twitter:bijection-core_2.10</exclude>
+ <exclude>com.twitter:bijection-avro_2.10</exclude>
+ <exclude>com.twitter:chill-protobuf</exclude>
+ <exclude>com.google.protobuf:protobuf-java</exclude>
+ <exclude>com.twitter:chill-thrift</exclude>
+ <exclude>org.apache.thrift:libthrift</exclude>
+ <exclude>commons-lang:commons-lang</exclude>
+ <exclude>junit:junit</exclude>
+ <exclude>de.javakaffee:kryo-serializers</exclude>
+ <exclude>joda-time:joda-time</exclude>
+ <exclude>org.apache.commons:commons-lang3</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>org.apache.commons:commons-math</exclude>
+ <exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>org.apache.httpcomponents:httpclient</exclude>
+ <exclude>org.apache.httpcomponents:httpcore</exclude>
+ <exclude>commons-codec:commons-codec</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-core</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
+ <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
+ <exclude>org.codehaus.jettison:jettison</exclude>
+ <exclude>stax:stax-api</exclude>
+ <exclude>com.typesafe:config</exclude>
+ <exclude>org.uncommons.maths:uncommons-maths</exclude>
+ <exclude>com.github.scopt:scopt_2.10</exclude>
+ <exclude>org.mortbay.jetty:servlet-api</exclude>
+ <exclude>commons-io:commons-io</exclude>
+ <exclude>commons-cli:commons-cli</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.flink:*</artifact>
+ <excludes>
+ <exclude>org/apache/flink/shaded/**</exclude>
+ <exclude>web-docs/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <!-- add Main-Class to manifest file -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>${package}.Job</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
</execution>
</executions>
</plugin>
@@ -185,4 +299,34 @@ under the License.
</plugin>
</plugins>
</build>
+ <profiles>
+ <profile>
+ <!-- A profile that does everyting correctly:
+ We set the Flink dependencies to provided -->
+ <id>build-jar</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-scala</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-core</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/04a73883/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cf042ed..3c6538e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -603,6 +603,7 @@ under the License.
<exclude>flink-staging/flink-avro/src/test/resources/avro/*.avsc</exclude>
<exclude>flink-staging/flink-avro/src/test/resources/testdata.avro</exclude>
<exclude>flink-staging/flink-avro/src/test/java/org/apache/flink/api/io/avro/generated/*.java</exclude>
+ <exclude>out/test/flink-avro/avro/user.avsc</exclude>
<!-- Configuration Files. -->
<exclude>**/flink-bin/conf/slaves</exclude>
<!-- Administrative files in the main trunk. -->