You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/02/28 10:57:21 UTC
[1/9] flink git commit: [FLINK-8762] [quickstarts] Make
'StreamingJob' the default main class and remove WordCount example from the
quickstart.
Repository: flink
Updated Branches:
refs/heads/release-1.4 392cfaaed -> 51231c471
[FLINK-8762] [quickstarts] Make 'StreamingJob' the default main class and remove WordCount example from the quickstart.
The packaged example jobs have been reported to not be terribly helpful and
simply create noise in the initial project setup.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/179cfdd4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/179cfdd4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/179cfdd4
Branch: refs/heads/release-1.4
Commit: 179cfdd4c6fd5f38ab01c2a55b17f251cc173dbc
Parents: 392cfaa
Author: Stephan Ewen <se...@apache.org>
Authored: Wed Feb 21 20:20:51 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:35:33 2018 +0100
----------------------------------------------------------------------
.../main/resources/archetype-resources/pom.xml | 6 +-
.../src/main/java/BatchJob.java | 20 +---
.../main/java/SocketTextStreamWordCount.java | 108 -------------------
.../src/main/java/StreamingJob.java | 21 ++--
.../src/main/java/WordCount.java | 94 ----------------
.../main/resources/archetype-resources/pom.xml | 6 +-
.../src/main/scala/BatchJob.scala | 25 ++---
.../main/scala/SocketTextStreamWordCount.scala | 69 ------------
.../src/main/scala/StreamingJob.scala | 24 ++---
.../src/main/scala/WordCount.scala | 53 ---------
10 files changed, 27 insertions(+), 399 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 71bc0a1..84584af 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -189,15 +189,11 @@ under the License.
</excludes>
</filter>
</filters>
- <!-- If you want to use ./bin/flink run <quickstart jar> uncomment the following lines.
- This will add a Main-Class entry to the manifest file -->
- <!--
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>${package}.StreamingJob</mainClass>
+ <mainClass>${package}.StreamingJob</mainClass>
</transformer>
</transformers>
- -->
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
index d0e68a4..9711924 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
@@ -23,22 +23,12 @@ import org.apache.flink.api.java.ExecutionEnvironment;
/**
* Skeleton for a Flink Batch Job.
*
- * <p>For a full example of a Flink Batch Job, see the WordCountJob.java file in the
- * same package/directory or have a look at the website.
+ * <p>For a tutorial how to write a Flink batch application, check the
+ * tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>.
*
- * <p>You can also generate a .jar file that you can submit on your Flink
- * cluster.
- * Just type
- * mvn clean package
- * in the projects root directory.
- * You will find the jar in
- * target/${artifactId}-${version}.jar
- * From the CLI you can then run
- * ./bin/flink run -c ${package}.BatchJob target/${artifactId}-${version}.jar
- *
- * <p>For more information on the CLI see:
- *
- * <p>http://flink.apache.org/docs/latest/apis/cli.html
+ * <p>To package your appliation into a JAR file for execution,
+ * change the main class in the POM.xml file to this class (simply search for 'mainClass')
+ * and run 'mvn clean package' on the command line.
*/
public class BatchJob {
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java
deleted file mode 100644
index 97df489..0000000
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/SocketTextStreamWordCount.java
+++ /dev/null
@@ -1,108 +0,0 @@
-package ${package};
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.Collector;
-
-/**
- * This example shows an implementation of WordCount with data from a text
- * socket. To run the example make sure that the service providing the text data
- * is already up and running.
- *
- * <p>To start an example socket text stream on your local machine run netcat from
- * a command line: <code>nc -lk 9999</code>, where the parameter specifies the
- * port number.
- *
- * <p>Usage:
- * <code>SocketTextStreamWordCount <hostname> <port></code>
- * <br>
- *
- * <p>This example shows how to:
- * <ul>
- * <li>use StreamExecutionEnvironment.socketTextStream
- * <li>write a simple Flink program
- * <li>write and use user-defined functions
- * </ul>
- *
- * @see <a href="www.openbsd.org/cgi-bin/man.cgi?query=nc">netcat</a>
- */
-public class SocketTextStreamWordCount {
-
- //
- // Program
- //
-
- public static void main(String[] args) throws Exception {
-
- if (args.length != 2){
- System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
- return;
- }
-
- String hostName = args[0];
- Integer port = Integer.parseInt(args[1]);
-
- // set up the execution environment
- final StreamExecutionEnvironment env = StreamExecutionEnvironment
- .getExecutionEnvironment();
-
- // get input data
- DataStream<String> text = env.socketTextStream(hostName, port);
-
- DataStream<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new LineSplitter())
- // group by the tuple field "0" and sum up tuple field "1"
- .keyBy(0)
- .sum(1);
-
- counts.print();
-
- // execute program
- env.execute("Java WordCount from SocketTextStream Example");
- }
-
- //
- // User Functions
- //
-
- /**
- * Implements the string tokenizer that splits sentences into words as a user-defined
- * FlatMapFunction. The function takes a line (String) and splits it into
- * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>).
- */
- public static final class LineSplitter implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
index 45a67ae..6027e75 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
@@ -20,26 +20,17 @@ package ${package};
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-
/**
* Skeleton for a Flink Streaming Job.
*
- * <p>For a full example of a Flink Streaming Job, see the SocketTextStreamWordCount.java
- * file in the same package/directory or have a look at the website.
- *
- * <p>You can also generate a .jar file that you can submit on your Flink
- * cluster.
- * Just type
- * mvn clean package
- * in the projects root directory.
- * You will find the jar in
- * target/${artifactId}-${version}.jar
- * From the CLI you can then run
- * ./bin/flink run -c ${package}.StreamingJob target/${artifactId}-${version}.jar
+ * <p>For a tutorial how to write a Flink streaming application, check the
+ * tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>.
*
- * <p>For more information on the CLI see:
+ * <p>To package your appliation into a JAR file for execution, run
+ * 'mvn clean package' on the command line.
*
- * <p>http://flink.apache.org/docs/latest/apis/cli.html
+ * <p>If you change the name of the main class (with the public static void main(String[] args))
+ * method, change the respective entry in the POM.xml file (simply search for 'mainClass').
*/
public class StreamingJob {
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java
deleted file mode 100644
index 6c95389..0000000
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/WordCount.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package ${package};
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.util.Collector;
-
-/**
- * Implements the "WordCount" program that computes a simple word occurrence histogram
- * over some sample data
- *
- * <p>This example shows how to:
- * <ul>
- * <li>write a simple Flink program.
- * <li>use Tuple data types.
- * <li>write and use user-defined functions.
- * </ul>
- *
- */
-public class WordCount {
-
- //
- // Program
- //
-
- public static void main(String[] args) throws Exception {
-
- // set up the execution environment
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // get input data
- DataSet<String> text = env.fromElements(
- "To be, or not to be,--that is the question:--",
- "Whether 'tis nobler in the mind to suffer",
- "The slings and arrows of outrageous fortune",
- "Or to take arms against a sea of troubles,"
- );
-
- DataSet<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new LineSplitter())
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0)
- .sum(1);
-
- // execute and print result
- counts.print();
-
- }
-
- //
- // User Functions
- //
-
- /**
- * Implements the string tokenizer that splits sentences into words as a user-defined
- * FlatMapFunction. The function takes a line (String) and splits it into
- * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>).
- */
- public static final class LineSplitter implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index b76f9b9..24b76f3 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -201,15 +201,11 @@ under the License.
</excludes>
</filter>
</filters>
- <!-- If you want to use ./bin/flink run <quickstart jar> uncomment the following lines.
- This will add a Main-Class entry to the manifest file -->
- <!--
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>${package}.StreamingJob</mainClass>
+ <mainClass>${package}.StreamingJob</mainClass>
</transformer>
</transformers>
- -->
</configuration>
</execution>
</executions>
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
index 4ecfeed..a533da9 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
@@ -23,26 +23,15 @@ import org.apache.flink.api.scala._
/**
* Skeleton for a Flink Batch Job.
*
- * For a full example of a Flink Batch Job, see the WordCountJob.scala file in the
- * same package/directory or have a look at the website.
+ * For a tutorial how to write a Flink batch application, check the
+ * tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>.
*
- * You can also generate a .jar file that you can submit on your Flink
- * cluster. Just type
- * {{{
- * mvn clean package
- * }}}
- * in the projects root directory. You will find the jar in
- * target/${artifactId}-${version}.jar
- * From the CLI you can then run
- * {{{
- * ./bin/flink run -c ${package}.BatchJob target/${artifactId}-${version}.jar
- * }}}
- *
- * For more information on the CLI see:
- *
- * http://flink.apache.org/docs/latest/apis/cli.html
+ * To package your appliation into a JAR file for execution,
+ * change the main class in the POM.xml file to this class (simply search for 'mainClass')
+ * and run 'mvn clean package' on the command line.
*/
object BatchJob {
+
def main(args: Array[String]) {
// set up the batch execution environment
val env = ExecutionEnvironment.getExecutionEnvironment
@@ -74,4 +63,4 @@ object BatchJob {
// execute program
env.execute("Flink Batch Scala API Skeleton")
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala
deleted file mode 100644
index a6987ac..0000000
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/SocketTextStreamWordCount.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-package ${package}
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.flink.streaming.api.scala._
-
-/**
- * This example shows an implementation of WordCount with data from a text socket.
- * To run the example make sure that the service providing the text data is already up and running.
- *
- * To start an example socket text stream on your local machine run netcat from a command line,
- * where the parameter specifies the port number:
- *
- * {{{
- * nc -lk 9999
- * }}}
- *
- * Usage:
- * {{{
- * SocketTextStreamWordCount <hostname> <port>
- * }}}
- *
- * This example shows how to:
- *
- * - use StreamExecutionEnvironment.socketTextStream
- * - write a simple Flink Streaming program in scala
- * - write and use user-defined functions
- */
-object SocketTextStreamWordCount {
-
- def main(args: Array[String]) {
- if (args.length != 2) {
- System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>")
- return
- }
-
- val hostName = args(0)
- val port = args(1).toInt
-
- val env = StreamExecutionEnvironment.getExecutionEnvironment
-
- // create streams for names and ages by mapping the inputs to the corresponding objects
- val text = env.socketTextStream(hostName, port)
- val counts = text.flatMap { _.toLowerCase.split("\\W+") filter { _.nonEmpty } }
- .map { (_, 1) }
- .keyBy(0)
- .sum(1)
-
- counts print
-
- env.execute("Scala WordCount from SocketTextStream Example")
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
index 7a45fc2..7c950b1 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
@@ -23,24 +23,14 @@ import org.apache.flink.streaming.api.scala._
/**
* Skeleton for a Flink Streaming Job.
*
- * For a full example of a Flink Streaming Job, see the SocketTextStreamWordCount.java
- * file in the same package/directory or have a look at the website.
+ * For a tutorial how to write a Flink streaming application, check the
+ * tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>.
*
- * You can also generate a .jar file that you can submit on your Flink
- * cluster. Just type
- * {{{
- * mvn clean package
- * }}}
- * in the projects root directory. You will find the jar in
- * target/${artifactId}-${version}.jar
- * From the CLI you can then run
- * {{{
- * ./bin/flink run -c ${package}.StreamingJob target/${artifactId}-${version}.jar
- * }}}
+ * To package your appliation into a JAR file for execution, run
+ * 'mvn clean package' on the command line.
*
- * For more information on the CLI see:
- *
- * http://flink.apache.org/docs/latest/apis/cli.html
+ * If you change the name of the main class (with the public static void main(String[] args))
+ * method, change the respective entry in the POM.xml file (simply search for 'mainClass').
*/
object StreamingJob {
def main(args: Array[String]) {
@@ -70,4 +60,4 @@ object StreamingJob {
// execute program
env.execute("Flink Streaming Scala API Skeleton")
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/179cfdd4/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala
deleted file mode 100644
index b88dcc6..0000000
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/WordCount.scala
+++ /dev/null
@@ -1,53 +0,0 @@
-package ${package}
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.flink.api.scala._
-
-/**
- * Implements the "WordCount" program that computes a simple word occurrence histogram
- * over some sample data
- *
- * This example shows how to:
- *
- * - write a simple Flink program.
- * - use Tuple data types.
- * - write and use user-defined functions.
- */
-object WordCount {
- def main(args: Array[String]) {
-
- // set up the execution environment
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- // get input data
- val text = env.fromElements("To be, or not to be,--that is the question:--",
- "Whether 'tis nobler in the mind to suffer", "The slings and arrows of outrageous fortune",
- "Or to take arms against a sea of troubles,")
-
- val counts = text.flatMap { _.toLowerCase.split("\\W+") }
- .map { (_, 1) }
- .groupBy(0)
- .sum(1)
-
- // execute and print result
- counts.print()
-
- }
-}
[8/9] flink git commit: [FLINK-8764] [docs] Adjust quickstart
documentation
Posted by se...@apache.org.
[FLINK-8764] [docs] Adjust quickstart documentation
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0c848aa9
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0c848aa9
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0c848aa9
Branch: refs/heads/release-1.4
Commit: 0c848aa98b466764873f2372af3ffd64cd8f71ac
Parents: 4eb0a0f
Author: Stephan Ewen <se...@apache.org>
Authored: Mon Feb 26 12:19:00 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:43:58 2018 +0100
----------------------------------------------------------------------
docs/quickstart/java_api_quickstart.md | 119 ++++++---------------------
docs/quickstart/scala_api_quickstart.md | 95 ++++++---------------
2 files changed, 52 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/0c848aa9/docs/quickstart/java_api_quickstart.md
----------------------------------------------------------------------
diff --git a/docs/quickstart/java_api_quickstart.md b/docs/quickstart/java_api_quickstart.md
index baf14de..9a32591 100644
--- a/docs/quickstart/java_api_quickstart.md
+++ b/docs/quickstart/java_api_quickstart.md
@@ -1,6 +1,6 @@
---
-title: "Sample Project using the Java API"
-nav-title: Sample Project in Java
+title: "Project Template for Java"
+nav-title: Project Template for Java
nav-parent_id: start
nav-pos: 0
---
@@ -86,120 +86,51 @@ quickstart/
│ └── myorg
│ └── quickstart
│ ├── BatchJob.java
- │ ├── SocketTextStreamWordCount.java
- │ ├── StreamingJob.java
- │ └── WordCount.java
+ │ └── StreamingJob.java
└── resources
└── log4j.properties
{% endhighlight %}
-The sample project is a __Maven project__, which contains four classes. _StreamingJob_ and _BatchJob_ are basic skeleton programs, _SocketTextStreamWordCount_ is a working streaming example and _WordCountJob_ is a working batch example. Please note that the _main_ method of all classes allow you to start Flink in a development/testing mode.
+The sample project is a __Maven project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program.
+The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments.
We recommend you __import this project into your IDE__ to develop and
-test it. If you use Eclipse, the [m2e plugin](http://www.eclipse.org/m2e/)
+test it. IntelliJ IDEA supports Maven projects out of the box.
+If you use Eclipse, the [m2e plugin](http://www.eclipse.org/m2e/)
allows to [import Maven projects](http://books.sonatype.com/m2eclipse-book/reference/creating-sect-importing-projects.html#fig-creating-import).
Some Eclipse bundles include that plugin by default, others require you
-to install it manually. The IntelliJ IDE supports Maven projects out of
-the box.
+to install it manually.
-
-*A note to Mac OS X users*: The default JVM heapsize for Java is too
+*A note to Mac OS X users*: The default JVM heapsize for Java mey be too
small for Flink. You have to manually increase it. In Eclipse, choose
`Run Configurations -> Arguments` and write into the `VM Arguments`
box: `-Xmx800m`.
## Build Project
-If you want to __build your project__, go to your project directory and
-issue the `mvn clean install -Pbuild-jar` command. You will
-__find a jar__ that runs on every Flink cluster with a compatible
-version, __target/original-your-artifact-id-your-version.jar__. There
-is also a fat-jar in __target/your-artifact-id-your-version.jar__ which,
-additionally, contains all dependencies that were added to the Maven
-project.
+If you want to __build/package your project__, go to your project directory and
+run the '`mvn clean package`' command.
+You will __find a JAR file__ that contains your application, plus connectors and libraries
+that you may have added as dependencoes to the application: `target/<artifact-id>-<version>.jar`.
+
+__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point,
+we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, the Flink
+can run time application from the JAR file without additionally specifying the main class.
## Next Steps
Write your application!
-The quickstart project contains a `WordCount` implementation, the
-"Hello World" of Big Data processing systems. The goal of `WordCount`
-is to determine the frequencies of words in a text, e.g., how often do
-the terms "the" or "house" occur in all Wikipedia texts.
-
-__Sample Input__:
-
-~~~bash
-big data is big
-~~~
-
-__Sample Output__:
-
-~~~bash
-big 2
-data 1
-is 1
-~~~
-
-The following code shows the `WordCount` implementation from the
-Quickstart which processes some text lines with two operators (a FlatMap
-and a Reduce operation via aggregating a sum), and prints the resulting
-words and counts to std-out.
-
-~~~java
-public class WordCount {
-
- public static void main(String[] args) throws Exception {
-
- // set up the execution environment
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- // get input data
- DataSet<String> text = env.fromElements(
- "To be, or not to be,--that is the question:--",
- "Whether 'tis nobler in the mind to suffer",
- "The slings and arrows of outrageous fortune",
- "Or to take arms against a sea of troubles,"
- );
-
- DataSet<Tuple2<String, Integer>> counts =
- // split up the lines in pairs (2-tuples) containing: (word,1)
- text.flatMap(new LineSplitter())
- // group by the tuple field "0" and sum up tuple field "1"
- .groupBy(0)
- .sum(1);
-
- // execute and print result
- counts.print();
- }
-}
-~~~
-
-The operations are defined by specialized classes, here the LineSplitter class.
-
-~~~java
-public static final class LineSplitter implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
- @Override
- public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
- // normalize and split the line
- String[] tokens = value.toLowerCase().split("\\W+");
-
- // emit the pairs
- for (String token : tokens) {
- if (token.length() > 0) {
- out.collect(new Tuple2<String, Integer>(token, 1));
- }
- }
- }
-}
-~~~
-
-{% gh_link /flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java "Check GitHub" %} for the full example code.
-
-For a complete overview over our API, have a look at the
+If you are writing a streaming application and you are looking for inspiration what to write,
+take a look at the [Stream Processing Application Tutorial]({{ site.baseurl }}/quickstart/run_example_quickstart.html#writing-a-flink-program)
+
+If you are writing a batch processing application and you are looking for inspiration what to write,
+take a look at the [Batch Application Examples]({{ site.baseurl }}/dev/batch/examples.html)
+
+For a complete overview over the APIa, have a look at the
[DataStream API]({{ site.baseurl }}/dev/datastream_api.html) and
[DataSet API]({{ site.baseurl }}/dev/batch/index.html) sections.
+
If you have any trouble, ask on our
[Mailing List](http://mail-archives.apache.org/mod_mbox/flink-user/).
We are happy to provide help.
http://git-wip-us.apache.org/repos/asf/flink/blob/0c848aa9/docs/quickstart/scala_api_quickstart.md
----------------------------------------------------------------------
diff --git a/docs/quickstart/scala_api_quickstart.md b/docs/quickstart/scala_api_quickstart.md
index 40c02a9..a7b73e3 100644
--- a/docs/quickstart/scala_api_quickstart.md
+++ b/docs/quickstart/scala_api_quickstart.md
@@ -1,6 +1,6 @@
---
-title: "Sample Project using the Scala API"
-nav-title: Sample Project in Scala
+title: "Project Template for Scala"
+nav-title: Project Template for Scala
nav-parent_id: start
nav-pos: 1
---
@@ -173,14 +173,18 @@ quickstart/
└── myorg
└── quickstart
├── BatchJob.scala
- ├── SocketTextStreamWordCount.scala
- ├── StreamingJob.scala
- └── WordCount.scala
+ └── StreamingJob.scala
{% endhighlight %}
-The sample project is a __Maven project__, which contains four classes. _StreamingJob_ and _BatchJob_ are basic skeleton programs, _SocketTextStreamWordCount_ is a working streaming example and _WordCountJob_ is a working batch example. Please note that the _main_ method of all classes allow you to start Flink in a development/testing mode.
+The sample project is a __Maven project__, which contains two classes: _StreamingJob_ and _BatchJob_ are the basic skeleton programs for a *DataStream* and *DataSet* program.
+The _main_ method is the entry point of the program, both for in-IDE testing/execution and for proper deployments.
-We recommend you __import this project into your IDE__. For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites:
+We recommend you __import this project into your IDE__.
+
+IntelliJ IDEA supports Maven out of the box and offers a plugin for Scala development.
+From our experience, IntelliJ provides the best experience for developing Flink applications.
+
+For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites:
* _Eclipse 4.x_
* [Scala IDE](http://download.scala-ide.org/sdk/lithium/e44/scala211/stable/site)
@@ -191,78 +195,33 @@ We recommend you __import this project into your IDE__. For Eclipse, you need th
* [m2eclipse-scala](http://alchim31.free.fr/m2e-scala/update-site)
* [Build Helper Maven Plugin](https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/)
-The IntelliJ IDE supports Maven out of the box and offers a plugin for
-Scala development.
+### Build Project
+If you want to __build/package your project__, go to your project directory and
+run the '`mvn clean package`' command.
+You will __find a JAR file__ that contains your application, plus connectors and libraries
+that you may have added as dependencoes to the application: `target/<artifact-id>-<version>.jar`.
-### Build Project
+__Note:__ If you use a different class than *StreamingJob* as the application's main class / entry point,
+we recommend you change the `mainClass` setting in the `pom.xml` file accordingly. That way, the Flink
+can run time application from the JAR file without additionally specifying the main class.
-If you want to __build your project__, go to your project directory and
-issue the `mvn clean package -Pbuild-jar` command. You will
-__find a jar__ that runs on every Flink cluster with a compatible
-version, __target/original-your-artifact-id-your-version.jar__. There
-is also a fat-jar in __target/your-artifact-id-your-version.jar__ which,
-additionally, contains all dependencies that were added to the Maven
-project.
## Next Steps
Write your application!
-The quickstart project contains a `WordCount` implementation, the
-"Hello World" of Big Data processing systems. The goal of `WordCount`
-is to determine the frequencies of words in a text, e.g., how often do
-the terms "the" or "house" occur in all Wikipedia texts.
-
-__Sample Input__:
-
-~~~bash
-big data is big
-~~~
+If you are writing a streaming application and you are looking for inspiration what to write,
+take a look at the [Stream Processing Application Tutorial]({{ site.baseurl }}/quickstart/run_example_quickstart.html#writing-a-flink-program)
-__Sample Output__:
-
-~~~bash
-big 2
-data 1
-is 1
-~~~
-
-The following code shows the `WordCount` implementation from the
-Quickstart which processes some text lines with two operators (a FlatMap
-and a Reduce operation via aggregating a sum), and prints the resulting
-words and counts to std-out.
-
-~~~scala
-object WordCountJob {
- def main(args: Array[String]) {
-
- // set up the execution environment
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- // get input data
- val text = env.fromElements("To be, or not to be,--that is the question:--",
- "Whether 'tis nobler in the mind to suffer", "The slings and arrows of outrageous fortune",
- "Or to take arms against a sea of troubles,")
-
- val counts = text.flatMap { _.toLowerCase.split("\\W+") }
- .map { (_, 1) }
- .groupBy(0)
- .sum(1)
-
- // emit result and print result
- counts.print()
- }
-}
-~~~
+If you are writing a batch processing application and you are looking for inspiration what to write,
+take a look at the [Batch Application Examples]({{ site.baseurl }}/dev/batch/examples.html)
-{% gh_link flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala "Check GitHub" %} for the full example code.
+For a complete overview over the APIa, have a look at the
+[DataStream API]({{ site.baseurl }}/dev/datastream_api.html) and
+[DataSet API]({{ site.baseurl }}/dev/batch/index.html) sections.
-For a complete overview over our API, have a look at the
-[DataStream API]({{ site.baseurl }}/dev/datastream_api.html),
-[DataSet API]({{ site.baseurl }}/dev/batch/index.html), and
-[Scala API Extensions]({{ site.baseurl }}/dev/scala_api_extensions.html)
-sections. If you have any trouble, ask on our
+If you have any trouble, ask on our
[Mailing List](http://mail-archives.apache.org/mod_mbox/flink-user/).
We are happy to provide help.
[3/9] flink git commit: [hotfix] [quickstarts] Fix block comments in
program stubs.
Posted by se...@apache.org.
[hotfix] [quickstarts] Fix block comments in program stubs.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/6bd35a3c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/6bd35a3c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/6bd35a3c
Branch: refs/heads/release-1.4
Commit: 6bd35a3c537eaa54c55312a6b3703208582b9229
Parents: 4a60fcb
Author: Stephan Ewen <se...@apache.org>
Authored: Wed Feb 21 21:05:23 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:36:08 2018 +0100
----------------------------------------------------------------------
.../main/resources/archetype-resources/src/main/java/BatchJob.java | 2 +-
.../resources/archetype-resources/src/main/java/StreamingJob.java | 2 +-
.../resources/archetype-resources/src/main/scala/BatchJob.scala | 2 +-
.../archetype-resources/src/main/scala/StreamingJob.scala | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/6bd35a3c/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
index 9515791..db2ee60 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
@@ -36,7 +36,7 @@ public class BatchJob {
// set up the batch execution environment
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
- /**
+ /*
* Here, you can start creating your execution plan for Flink.
*
* Start with getting some data from the environment, like
http://git-wip-us.apache.org/repos/asf/flink/blob/6bd35a3c/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
index 4091889..5bcee21 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
@@ -38,7 +38,7 @@ public class StreamingJob {
// set up the streaming execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- /**
+ /*
* Here, you can start creating your execution plan for Flink.
*
* Start with getting some data from the environment, like
http://git-wip-us.apache.org/repos/asf/flink/blob/6bd35a3c/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
index 46520b7..329e6c2 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
@@ -36,7 +36,7 @@ object BatchJob {
// set up the batch execution environment
val env = ExecutionEnvironment.getExecutionEnvironment
- /**
+ /*
* Here, you can start creating your execution plan for Flink.
*
* Start with getting some data from the environment, like
http://git-wip-us.apache.org/repos/asf/flink/blob/6bd35a3c/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
index 20115a7..bab0bb9 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
@@ -37,7 +37,7 @@ object StreamingJob {
// set up the streaming execution environment
val env = StreamExecutionEnvironment.getExecutionEnvironment
- /**
+ /*
* Here, you can start creating your execution plan for Flink.
*
* Start with getting some data from the environment, like
[5/9] flink git commit: [FLINK-8765] [quickstarts] Simplify
quickstart properties
Posted by se...@apache.org.
[FLINK-8765] [quickstarts] Simplify quickstart properties
This does not pull out the slf4j and log4j version into properties any more,
making the quickstarts a bit simpler.
Given that both versions are used only once, and only for the feature to have
convenience logging in the IDE, the versions might as well be defined directly
in the dependencies.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/11e19a56
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/11e19a56
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/11e19a56
Branch: refs/heads/release-1.4
Commit: 11e19a56bfdedb4f41ecc3dcf1cd4120191cdb1a
Parents: fa30638
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Feb 23 11:10:43 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:43:01 2018 +0100
----------------------------------------------------------------------
.../src/main/resources/archetype-resources/pom.xml | 6 ++----
.../src/main/resources/archetype-resources/pom.xml | 6 ++----
2 files changed, 4 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/11e19a56/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 6ac05b0..50b0351 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -31,8 +31,6 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>@project.version@</flink.version>
- <slf4j.version>@slf4j.version@</slf4j.version>
- <log4j.version>@log4j.version@</log4j.version>
<scala.binary.version>@scala.binary.version@</scala.binary.version>
</properties>
@@ -82,13 +80,13 @@ under the License.
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
+ <version>1.7.7</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <version>1.2.17</version>
<scope>runtime</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/flink/blob/11e19a56/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index b318a19..4a866c3 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -46,8 +46,6 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>@project.version@</flink.version>
- <slf4j.version>@slf4j.version@</slf4j.version>
- <log4j.version>@log4j.version@</log4j.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.11</scala.version>
</properties>
@@ -92,13 +90,13 @@ under the License.
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
+ <version>1.7.7</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <version>1.2.17</version>
<scope>runtime</scope>
</dependency>
</dependencies>
[4/9] flink git commit: [FLINK-8764] [quickstarts] Make quickstarts
work out of the box for IDE and JAR packaging
Posted by se...@apache.org.
[FLINK-8764] [quickstarts] Make quickstarts work out of the box for IDE and JAR packaging
- All Flink and Scala dependencies are properly set to provided
- That way, Maven JAR packaging behaves correctly by default
- Eclipse adds 'provided' dependencies to the classpath when running programs, so works out of the box
- There is a profile that automatically activates in IntelliJ that adds the necessary
dependencies in 'compile' scope to make it run out of the box.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fa306385
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fa306385
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fa306385
Branch: refs/heads/release-1.4
Commit: fa306385fe190450251fd902a544d5d3fa2ca664
Parents: 6bd35a3
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Feb 23 10:53:22 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:42:33 2018 +0100
----------------------------------------------------------------------
.../main/resources/archetype-resources/pom.xml | 231 ++++++++---------
.../main/resources/archetype-resources/pom.xml | 246 ++++++++-----------
2 files changed, 207 insertions(+), 270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/fa306385/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 84584af..6ac05b0 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -50,161 +50,53 @@ under the License.
</repository>
</repositories>
- <!--
-
- Execute "mvn clean package -Pbuild-jar"
- to build a jar file out of this project!
-
- How to use the Flink Quickstart pom:
-
- a) Adding new dependencies:
- You can add dependencies to the list below.
-
- b) Build a jar for running on the cluster:
-
- "mvn clean package -Pbuild-jar"
- This will create a fat-jar which contains all dependencies necessary for running the created jar in a cluster.
- -->
-
<dependencies>
<!-- Apache Flink dependencies -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${flink.version}</version>
- </dependency>
+ <!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
- <!-- This dependency is required to actually execute jobs. It is currently pulled in by
- flink-streaming-java, but we explicitly depend on it to safeguard against future changes. -->
<groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_${scala.binary.version}</artifactId>
+ <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
+ <scope>provided</scope>
</dependency>
+
+ <!-- Add connector dependencies here. They must be in the default scope (compile). -->
+
+ <!-- Example:
+
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+ <artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
</dependency>
+ -->
- <!-- explicitly add a standard logging framework, as Flink does not have
- a hard dependency on one specific framework by default -->
+ <!-- Add logging framework, to produce console output when running in the IDE. -->
+ <!-- These dependencies are excluded from the application JAR by default. -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
+ <scope>runtime</scope>
</dependency>
</dependencies>
- <profiles>
- <profile>
- <!-- Profile for packaging correct JAR files -->
- <id>build-jar</id>
-
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
- except flink and its transitive dependencies. The resulting fat-jar can be executed
- on a cluster. Change the value of Program-Class if your program entry point changes. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.4.1</version>
- <executions>
- <!-- Run shade goal on package phase -->
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <excludes>
- <exclude>org.apache.flink:force-shading</exclude>
- <exclude>com.google.code.findbugs:jsr305</exclude>
- <exclude>org.slf4j:*</exclude>
- </excludes>
- </artifactSet>
- <filters>
- <filter>
- <!-- Do not copy the signatures in the META-INF folder.
- Otherwise, this might cause SecurityExceptions when using the JAR. -->
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>${package}.StreamingJob</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
<build>
<plugins>
+
+ <!-- Java Compiler -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
@@ -214,12 +106,57 @@ under the License.
<target>1.8</target>
</configuration>
</plugin>
+
+ <!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
+ <!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.0.0</version>
+ <executions>
+ <!-- Run shade goal on package phase -->
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <exclude>org.apache.flink:force-shading</exclude>
+ <exclude>com.google.code.findbugs:jsr305</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>log4j:*</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <!-- Do not copy the signatures in the META-INF folder.
+ Otherwise, this might cause SecurityExceptions when using the JAR. -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>${package}.StreamingJob</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
- <!-- If you want to use Java 8 Lambda Expressions uncomment the following lines -->
- <!--
<pluginManagement>
<plugins>
+
+ <!-- If you want to use Java 8 Lambda Expressions uncomment the following lines -->
+ <!--
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
@@ -235,7 +172,9 @@ under the License.
</dependency>
</dependencies>
</plugin>
+ -->
+ <!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
@@ -246,10 +185,10 @@ under the License.
<pluginExecution>
<pluginExecutionFilter>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <versionRange>[2.4,)</versionRange>
+ <artifactId>maven-shade-plugin</artifactId>
+ <versionRange>[3.0.0,)</versionRange>
<goals>
- <goal>single</goal>
+ <goal>shade</goal>
</goals>
</pluginExecutionFilter>
<action>
@@ -276,6 +215,36 @@ under the License.
</plugin>
</plugins>
</pluginManagement>
- -->
</build>
+
+ <!-- This profile helps to make things run out of the box in IntelliJ -->
+ <!-- Its adds Flink's core classes to the runtime class path. -->
+ <!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
+ <profiles>
+ <profile>
+ <id>add-dependencies-for-IDEA</id>
+
+ <activation>
+ <property>
+ <name>idea.version</name>
+ </property>
+ </activation>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>${flink.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
http://git-wip-us.apache.org/repos/asf/flink/blob/fa306385/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
index 24b76f3..b318a19 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml
@@ -52,171 +52,104 @@ under the License.
<scala.version>2.11.11</scala.version>
</properties>
- <!--
-
- Execute "mvn clean package -Pbuild-jar"
- to build a jar file out of this project!
-
- How to use the Flink Quickstart pom:
-
- a) Adding new dependencies:
- You can add dependencies to the list below.
-
- b) Build a jar for running on the cluster:
-
- "mvn clean package -Pbuild-jar"
- This will create a fat-jar which contains all dependencies necessary for running the created jar in a cluster.
- -->
-
<dependencies>
<!-- Apache Flink dependencies -->
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${flink.version}</version>
- </dependency>
- <dependency>
- <!-- This dependency is required to actually execute jobs. It is currently pulled in by
- flink-streaming-java, but we explicitly depend on it to safeguard against future changes. -->
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- </dependency>
+ <!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
+ <scope>provided</scope>
</dependency>
+ <!-- Scala Library, provided by Flink as well. -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Add connector dependencies here. They must be in the default scope (compile). -->
+
+ <!-- Example:
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
</dependency>
+ -->
- <!-- explicitly add a standard logging framework, as Flink does not have
- a hard dependency on one specific framework by default -->
+ <!-- Add logging framework, to produce console output when running in the IDE. -->
+ <!-- These dependencies are excluded from the application JAR by default. -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
+ <scope>runtime</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
+ <scope>runtime</scope>
</dependency>
</dependencies>
- <profiles>
- <profile>
- <!-- Profile for packaging correct JAR files -->
- <id>build-jar</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-core</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-scala_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <!-- We use the maven-shade plugin to create a fat jar that contains all dependencies
- except flink and its transitive dependencies. The resulting fat-jar can be executed
- on a cluster. Change the value of Program-Class if your program entry point changes. -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.4.1</version>
- <executions>
- <!-- Run shade goal on package phase -->
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <excludes>
- <exclude>org.apache.flink:force-shading</exclude>
- <exclude>com.google.code.findbugs:jsr305</exclude>
- <exclude>org.slf4j:*</exclude>
- </excludes>
- </artifactSet>
- <filters>
- <filter>
- <!-- Do not copy the signatures in the META-INF folder.
- Otherwise, this might cause SecurityExceptions when using the JAR. -->
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>${package}.StreamingJob</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
-
<build>
<plugins>
+ <!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
+ <!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.0.0</version>
+ <executions>
+ <!-- Run shade goal on package phase -->
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <exclude>org.apache.flink:force-shading</exclude>
+ <exclude>com.google.code.findbugs:jsr305</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>log4j:*</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <!-- Do not copy the signatures in the META-INF folder.
+ Otherwise, this might cause SecurityExceptions when using the JAR. -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>${package}.StreamingJob</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!-- Java Compiler -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
@@ -226,6 +159,8 @@ under the License.
<target>1.8</target>
</configuration>
</plugin>
+
+ <!-- Scala Compiler -->
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
@@ -240,7 +175,7 @@ under the License.
</executions>
</plugin>
- <!-- Eclipse Integration -->
+ <!-- Eclipse Scala Integration -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
@@ -255,10 +190,8 @@ under the License.
<buildcommand>org.scala-ide.sdt.core.scalabuilder</buildcommand>
</buildcommands>
<classpathContainers>
- <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER
- </classpathContainer>
- <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER
- </classpathContainer>
+ <classpathContainer>org.scala-ide.sdt.launching.SCALA_CONTAINER</classpathContainer>
+ <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
</classpathContainers>
<excludes>
<exclude>org.scala-lang:scala-library</exclude>
@@ -270,8 +203,6 @@ under the License.
</sourceIncludes>
</configuration>
</plugin>
-
- <!-- Adding scala source directories to build path -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
@@ -307,4 +238,41 @@ under the License.
</plugin>
</plugins>
</build>
+
+ <!-- This profile helps to make things run out of the box in IntelliJ -->
+ <!-- Its adds Flink's core classes to the runtime class path. -->
+ <!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
+ <profiles>
+ <profile>
+ <id>add-dependencies-for-IDEA</id>
+
+ <activation>
+ <property>
+ <name>idea.version</name>
+ </property>
+ </activation>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-scala_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
[2/9] flink git commit: [hotfix] [quickstarts] Fix header and package
declaration order.
Posted by se...@apache.org.
[hotfix] [quickstarts] Fix header and package declaration order.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4a60fcb0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4a60fcb0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4a60fcb0
Branch: refs/heads/release-1.4
Commit: 4a60fcb0f41e96221108a43fc7037e08ea047758
Parents: 179cfdd
Author: Stephan Ewen <se...@apache.org>
Authored: Wed Feb 21 20:30:35 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:35:48 2018 +0100
----------------------------------------------------------------------
.../resources/archetype-resources/src/main/java/BatchJob.java | 6 +++---
.../archetype-resources/src/main/java/StreamingJob.java | 6 +++---
.../archetype-resources/src/main/scala/BatchJob.scala | 6 +++---
.../archetype-resources/src/main/scala/StreamingJob.scala | 6 +++---
4 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4a60fcb0/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
index 9711924..9515791 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/BatchJob.java
@@ -1,6 +1,4 @@
-package ${package};
-
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,6 +16,8 @@ package ${package};
* limitations under the License.
*/
+package ${package};
+
import org.apache.flink.api.java.ExecutionEnvironment;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/4a60fcb0/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
index 6027e75..4091889 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/src/main/java/StreamingJob.java
@@ -1,6 +1,4 @@
-package ${package};
-
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,6 +16,8 @@ package ${package};
* limitations under the License.
*/
+package ${package};
+
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/4a60fcb0/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
index a533da9..46520b7 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/BatchJob.scala
@@ -1,6 +1,4 @@
-package ${package}
-
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,6 +16,8 @@ package ${package}
* limitations under the License.
*/
+package ${package}
+
import org.apache.flink.api.scala._
/**
http://git-wip-us.apache.org/repos/asf/flink/blob/4a60fcb0/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
index 7c950b1..20115a7 100644
--- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
+++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/src/main/scala/StreamingJob.scala
@@ -1,6 +1,4 @@
-package ${package}
-
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,6 +16,8 @@ package ${package}
* limitations under the License.
*/
+package ${package}
+
import org.apache.flink.streaming.api.scala._
/**
[7/9] flink git commit: [FLINK-8767] [quickstarts] Set the
maven.compiler.source and .target properties for Java Quickstart
Posted by se...@apache.org.
[FLINK-8767] [quickstarts] Set the maven.compiler.source and .target properties for Java Quickstart
Setting these properties helps properly pinning the Java version in IntelliJ.
Without these properties, Java version keeps switching back to 1.5 in some setups.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4eb0a0fb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4eb0a0fb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4eb0a0fb
Branch: refs/heads/release-1.4
Commit: 4eb0a0fb9f41b29f6b87ce77f5e985f2b9fa30a3
Parents: 23c37cf
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Feb 23 11:18:36 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:43:35 2018 +0100
----------------------------------------------------------------------
.../src/main/resources/archetype-resources/pom.xml | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4eb0a0fb/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index d53415a..0ca6eb9 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -31,7 +31,10 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>@project.version@</flink.version>
+ <java.version>1.8</java.version>
<scala.binary.version>2.11</scala.binary.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
</properties>
<repositories>
@@ -100,8 +103,8 @@ under the License.
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
- <source>1.8</source>
- <target>1.8</target>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
</configuration>
</plugin>
@@ -158,8 +161,8 @@ under the License.
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <source>1.8</source>
- <target>1.8</target>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
<compilerId>jdt</compilerId>
</configuration>
<dependencies>
[6/9] flink git commit: [FLINK-8766] [quickstarts] Pin scala runtime
version for Java Quickstart
Posted by se...@apache.org.
[FLINK-8766] [quickstarts] Pin scala runtime version for Java Quickstart
Followup to FLINK-7414, which pinned the scala version for the Scala Quickstart
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/23c37cf4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/23c37cf4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/23c37cf4
Branch: refs/heads/release-1.4
Commit: 23c37cf43cac94ecaa43641d264f7422bc1c57ed
Parents: 11e19a5
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Feb 23 11:13:01 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:43:19 2018 +0100
----------------------------------------------------------------------
.../src/main/resources/archetype-resources/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/23c37cf4/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
----------------------------------------------------------------------
diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
index 50b0351..d53415a 100644
--- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
+++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml
@@ -31,7 +31,7 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>@project.version@</flink.version>
- <scala.binary.version>@scala.binary.version@</scala.binary.version>
+ <scala.binary.version>2.11</scala.binary.version>
</properties>
<repositories>
[9/9] flink git commit: [FLINK-8791] [docs] Fix documentation about
configuring dependencies
Posted by se...@apache.org.
[FLINK-8791] [docs] Fix documentation about configuring dependencies
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/51231c47
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/51231c47
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/51231c47
Branch: refs/heads/release-1.4
Commit: 51231c471bca5e400e1bf152d3cfbdb68c809567
Parents: 0c848aa
Author: Stephan Ewen <se...@apache.org>
Authored: Mon Feb 26 16:41:24 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 28 11:44:49 2018 +0100
----------------------------------------------------------------------
docs/dev/linking.md | 96 --------
docs/dev/linking_with_flink.md | 146 -----------
docs/redirects/linking_with_flink.md | 25 ++
docs/redirects/linking_with_optional_modules.md | 25 ++
docs/start/dependencies.md | 244 +++++++++++++++++++
5 files changed, 294 insertions(+), 242 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/51231c47/docs/dev/linking.md
----------------------------------------------------------------------
diff --git a/docs/dev/linking.md b/docs/dev/linking.md
deleted file mode 100644
index 78ef544..0000000
--- a/docs/dev/linking.md
+++ /dev/null
@@ -1,96 +0,0 @@
----
-nav-title: "Linking with Optional Modules"
-title: "Linking with modules not contained in the binary distribution"
-nav-parent_id: start
-nav-pos: 10
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-The binary distribution contains jar packages in the `lib` folder that are automatically
-provided to the classpath of your distributed programs. Almost all of Flink classes are
-located there with a few exceptions, for example the streaming connectors and some freshly
-added modules. To run code depending on these modules you need to make them accessible
-during runtime, for which we suggest two options:
-
-1. Either copy the required jar files to the `lib` folder onto all of your TaskManagers.
-Note that you have to restart your TaskManagers after this.
-2. Or package them with your code.
-
-The latter version is recommended as it respects the classloader management in Flink.
-
-### Packaging dependencies with your usercode with Maven
-
-To provide these dependencies not included by Flink we suggest two options with Maven.
-
-1. The maven assembly plugin builds a so-called uber-jar (executable jar) containing all your dependencies.
-The assembly configuration is straight-forward, but the resulting jar might become bulky.
-See [maven-assembly-plugin](http://maven.apache.org/plugins/maven-assembly-plugin/usage.html) for further information.
-2. The maven unpack plugin unpacks the relevant parts of the dependencies and
-then packages it with your code.
-
-Using the latter approach in order to bundle the Kafka connector, `flink-connector-kafka`
-you would need to add the classes from both the connector and the Kafka API itself. Add
-the following to your plugins section.
-
-~~~xml
-<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.9</version>
- <executions>
- <execution>
- <id>unpack</id>
- <!-- executed just before the package phase -->
- <phase>prepare-package</phase>
- <goals>
- <goal>unpack</goal>
- </goals>
- <configuration>
- <artifactItems>
- <!-- For Flink connector classes -->
- <artifactItem>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-kafka</artifactId>
- <version>{{ site.version }}</version>
- <type>jar</type>
- <overWrite>false</overWrite>
- <outputDirectory>${project.build.directory}/classes</outputDirectory>
- <includes>org/apache/flink/**</includes>
- </artifactItem>
- <!-- For Kafka API classes -->
- <artifactItem>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_<YOUR_SCALA_VERSION></artifactId>
- <version><YOUR_KAFKA_VERSION></version>
- <type>jar</type>
- <overWrite>false</overWrite>
- <outputDirectory>${project.build.directory}/classes</outputDirectory>
- <includes>kafka/**</includes>
- </artifactItem>
- </artifactItems>
- </configuration>
- </execution>
- </executions>
-</plugin>
-~~~
-
-Now when running `mvn clean package` the produced jar includes the required dependencies.
-
-{% top %}
http://git-wip-us.apache.org/repos/asf/flink/blob/51231c47/docs/dev/linking_with_flink.md
----------------------------------------------------------------------
diff --git a/docs/dev/linking_with_flink.md b/docs/dev/linking_with_flink.md
deleted file mode 100644
index f2380b2..0000000
--- a/docs/dev/linking_with_flink.md
+++ /dev/null
@@ -1,146 +0,0 @@
----
-title: "Linking with Flink"
-nav-parent_id: start
-nav-pos: 2
----
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-
-To write programs with Flink, you need to include the Flink library corresponding to
-your programming language in your project.
-
-The simplest way to do this is to use one of the quickstart scripts: either for
-[Java]({{ site.baseurl }}/quickstart/java_api_quickstart.html) or for [Scala]({{ site.baseurl }}/quickstart/scala_api_quickstart.html). They
-create a blank project from a template (a Maven Archetype), which sets up everything for you. To
-manually create the project, you can use the archetype and create a project by calling:
-
-<div class="codetabs" markdown="1">
-<div data-lang="java" markdown="1">
-{% highlight bash %}
-mvn archetype:generate \
- -DarchetypeGroupId=org.apache.flink \
- -DarchetypeArtifactId=flink-quickstart-java \
- -DarchetypeVersion={{site.version }}
-{% endhighlight %}
-</div>
-<div data-lang="scala" markdown="1">
-{% highlight bash %}
-mvn archetype:generate \
- -DarchetypeGroupId=org.apache.flink \
- -DarchetypeArtifactId=flink-quickstart-scala \
- -DarchetypeVersion={{site.version }}
-{% endhighlight %}
-</div>
-</div>
-
-The archetypes are working for stable releases and preview versions (`-SNAPSHOT`).
-
-If you want to add Flink to an existing Maven project, add the following entry to your
-*dependencies* section in the *pom.xml* file of your project:
-
-<div class="codetabs" markdown="1">
-<div data-lang="java" markdown="1">
-{% highlight xml %}
-<!-- Use this dependency if you are using the DataStream API -->
-<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java{{ site.scala_version_suffix }}</artifactId>
- <version>{{site.version }}</version>
-</dependency>
-<!-- Use this dependency if you are using the DataSet API -->
-<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-java</artifactId>
- <version>{{site.version }}</version>
-</dependency>
-<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients{{ site.scala_version_suffix }}</artifactId>
- <version>{{site.version }}</version>
-</dependency>
-{% endhighlight %}
-</div>
-<div data-lang="scala" markdown="1">
-{% highlight xml %}
-<!-- Use this dependency if you are using the DataStream API -->
-<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-scala{{ site.scala_version_suffix }}</artifactId>
- <version>{{site.version }}</version>
-</dependency>
-<!-- Use this dependency if you are using the DataSet API -->
-<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-scala{{ site.scala_version_suffix }}</artifactId>
- <version>{{site.version }}</version>
-</dependency>
-<dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients{{ site.scala_version_suffix }}</artifactId>
- <version>{{site.version }}</version>
-</dependency>
-{% endhighlight %}
-
-**Important:** When working with the Scala API you must have one of these two imports:
-{% highlight scala %}
-import org.apache.flink.api.scala._
-{% endhighlight %}
-
-or
-
-{% highlight scala %}
-import org.apache.flink.api.scala.createTypeInformation
-{% endhighlight %}
-
-The reason is that Flink analyzes the types that are used in a program and generates serializers
-and comparators for them. By having either of those imports you enable an implicit conversion
-that creates the type information for Flink operations.
-
-If you would rather use SBT, see [here]({{ site.baseurl }}/quickstart/scala_api_quickstart.html#sbt).
-</div>
-</div>
-
-#### Scala Dependency Versions
-
-Because Scala 2.10 binary is not compatible with Scala 2.11 binary, we provide multiple artifacts
-to support both Scala versions.
-
-Starting from the 0.10 line, we cross-build all Flink modules for both 2.10 and 2.11. If you want
-to run your program on Flink with Scala 2.11, you need to add a `_2.11` suffix to the `artifactId`
-values of the Flink modules in your dependencies section.
-
-If you are looking for building Flink with Scala 2.11, please check
-[build guide]({{ site.baseurl }}/start/building.html#scala-versions).
-
-#### Hadoop Dependency Versions
-
-If you are using Flink together with Hadoop, the version of the dependency may vary depending on the
-version of Hadoop (or more specifically, HDFS) that you want to use Flink with. Please refer to the
-[downloads page](http://flink.apache.org/downloads.html) for a list of available versions, and instructions
-on how to link with custom versions of Hadoop.
-
-In order to link against the latest SNAPSHOT versions of the code, please follow
-[this guide](http://flink.apache.org/how-to-contribute.html#snapshots-nightly-builds).
-
-The *flink-clients* dependency is only necessary to invoke the Flink program locally (for example to
-run it standalone for testing and debugging). If you intend to only export the program as a JAR
-file and [run it on a cluster]({{ site.baseurl }}/dev/cluster_execution.html), you can skip that dependency.
-
-{% top %}
-
http://git-wip-us.apache.org/repos/asf/flink/blob/51231c47/docs/redirects/linking_with_flink.md
----------------------------------------------------------------------
diff --git a/docs/redirects/linking_with_flink.md b/docs/redirects/linking_with_flink.md
new file mode 100644
index 0000000..1289487
--- /dev/null
+++ b/docs/redirects/linking_with_flink.md
@@ -0,0 +1,25 @@
+---
+title: "Linking with Flink"
+layout: redirect
+redirect: /start/dependencies.html
+permalink: /dev/linking_with_flink.html
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
http://git-wip-us.apache.org/repos/asf/flink/blob/51231c47/docs/redirects/linking_with_optional_modules.md
----------------------------------------------------------------------
diff --git a/docs/redirects/linking_with_optional_modules.md b/docs/redirects/linking_with_optional_modules.md
new file mode 100644
index 0000000..e494fbc
--- /dev/null
+++ b/docs/redirects/linking_with_optional_modules.md
@@ -0,0 +1,25 @@
+---
+title: "Linking with Optional Modules"
+layout: redirect
+redirect: /start/dependencies.html
+permalink: /dev/linking.html
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
http://git-wip-us.apache.org/repos/asf/flink/blob/51231c47/docs/start/dependencies.md
----------------------------------------------------------------------
diff --git a/docs/start/dependencies.md b/docs/start/dependencies.md
new file mode 100644
index 0000000..1375c6f
--- /dev/null
+++ b/docs/start/dependencies.md
@@ -0,0 +1,244 @@
+---
+title: "Configuring Dependencies, Connectors, Libraries"
+nav-parent_id: start
+nav-pos: 2
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Every Flink application depends on a set of Flink libraries. At the bare minimum, the application depends
+on the Flink APIs. Many applications depend in addition on certain connector libraries (like Kafka, Cassandra, etc.).
+When running Flink applications (either in a distributed deployment, or in the IDE for testing), the Flink
+runtime library must be available as well.
+
+
+## Flink Core and Application Dependencies
+
+As with most systems that run user-defined applications, there are two broad categories of dependencies and libraries in Flink:
+
+ - **Flink Core Dependenies**: Flink itself consists of a set of classes and dependencies that are needed to run the system, for example
+ coordination, networking, checkpoints, failover, APIs, operations (such as windowing), resource management, etc.
+ The set of all these classes and dependencies forms the core of Flink's runtime and must be present when a Flink
+ application is started.
+
+ These core classes and dependencies are packaged in the `flink-dist` jar. They are part of Flink's `lib` folder and
+ part of the basic Flink container images. Think of these dependencies as similar to Java's core library (`rt.jar`, `charsets.jar`, etc.),
+ which contains the classes like `String` and `List`.
+
+ The Flink Core Dependencies do not contain any connectors or libraries (CEP, SQL, ML, etc.) in order to avoid having an excessive
+ number of dependencies and classes in the classpath by default. In fact, we try to keep the core dependencies as slim as possible
+ to keep the default classpath small and avoid dependency clashes.
+
+ - The **User Application Dependencies** are all connectors, formats, or libraries that a specific user application needs.
+
+ The user application is typically packaged into an *application jar*, which contains the application code and the required
+ connector and library dependencies.
+
+ The user application dependencies explicitly do not include the Flink DataSet / DataStream APIs and runtime dependencies,
+ because those are already part of Flink's Core Dependencies.
+
+
+## Setting up a Project: Basic Dependencies
+
+Every Flink application needs as the bare minimum the API dependencies, to develop against.
+For Maven, you can use the [Java Project Template]({{ site.baseurl }}/quickstart/java_api_quickstart.html)
+or [Scala Project Template]({{ site.baseurl }}/quickstart/scala_api_quickstart.html) to create
+a program skeleton with these initial dependencies.
+
+When setting up a project manually, you need to add the following dependencies for the Java/Scala API
+(here presented in Maven syntax, but the same dependencies apply to other build tools (Gradle, SBT, etc.) as well.
+
+<div class="codetabs" markdown="1">
+<div data-lang="java" markdown="1">
+{% highlight xml %}
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-java</artifactId>
+ <version>{{site.version }}</version>
+ <scope>provided</scope>
+</dependency>
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java{{ site.scala_version_suffix }}</artifactId>
+ <version>{{site.version }}</version>
+ <scope>provided</scope>
+</dependency>
+{% endhighlight %}
+</div>
+<div data-lang="scala" markdown="1">
+{% highlight xml %}
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-scala{{ site.scala_version_suffix }}</artifactId>
+ <version>{{site.version }}</version>
+ <scope>provided</scope>
+</dependency>
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-scala{{ site.scala_version_suffix }}</artifactId>
+ <version>{{site.version }}</version>
+ <scope>provided</scope>
+</dependency>
+{% endhighlight %}
+</div>
+</div>
+
+**Important:** Please note that all these dependencies have their scope set to *provided*.
+That means that they are needed to compile against, but that they should not be packaged into the
+project's resulting application jar file - these dependencies are Flink Core Dependencies,
+which are already available in any setup.
+
+It is highly recommended to keep the dependencies in scope *provided*. If they are not set to *provided*,
+the best case is that the resulting JAR becomes excessively large, because it also contains all Flink core
+dependencies. The worst case is that the Flink core dependencies that are added to the application's jar file
+clash with some of your own dependency versions (which is normally avoided through inverted classloading).
+
+**Note on IntelliJ:** To make the applications run within IntelliJ IDEA, the Flink dependencies need
+to be declared in scope *compile* rather than *provided*. Otherwise IntelliJ will not add them to the classpath and
+the in-IDE execution will fail with a `NoClassDefFountError`. To avoid having to declare the
+dependency scope as *compile* (which is not recommended, see above), the above linked Java- and Scala
+project templates use a trick: They add a profile that selectively activates when the application
+is run in IntelliJ and only then promotes the dependencies to scope *compile*, without affecting
+the packaging of the JAR files.
+
+
+## Adding Connector and Library Dependencies
+
+Most applications need specific connectors or libraries to run, for example a connector to Kafka, Cassandra, etc.
+These connectors are not part of Flink's core dependencies and must hence be added as dependencies to the application
+
+Below is an example adding the connector for Kafka 0.10 as a dependency (Maven syntax):
+{% highlight xml %}
+<dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-kafka-0.10{{ site.scala_version_suffix }}</artifactId>
+ <version>{{site.version }}</version>
+</dependency>
+{% endhighlight %}
+
+We recommend to package the application code and all its required dependencies into one *jar-with-dependencies* which
+we refer to as the *application jar*. The application jar can be submitted to an already running Flink cluster,
+or added to a Flink application container image.
+
+Projects created from the [Java Project Template]({{ site.baseurl }}/quickstart/java_api_quickstart.html) or
+[Scala Project Template]({{ site.baseurl }}/quickstart/scala_api_quickstart.html) are configured to automatically include
+the application dependencies into the application jar when running `mvn clean package`. For projects that are
+not set up from those templates, we recommend to add the Maven Shade Plugin (as listed in the Appendix below)
+to build the application jar with all required dependencies.
+
+**Important:** For Maven (and other build tools) to correctly package the dependencies into the application jar,
+these application dependencies must be specified in scope *compile* (unlike the core dependencies, which
+must be specified in scope *provided*).
+
+
+## Scala Versions
+
+Scala versions (2.10, 2.11, 2.12, etc.) are not binary compatible with one another.
+For that reason, Flink for Scala 2.11 cannot be used with an application that uses
+Scala 2.12.
+
+All Flink dependencies that (transitively) depend on Scala are suffixed with the
+Scala version that they are built for, for example `flink-streaming-scala_2.11`.
+
+Developers that only use Java can pick any Scala version, Scala developers need to
+pick the Scala version that matches their application's Scala version.
+
+Please refer to the [build guide]({{ site.baseurl }}/start/building.html#scala-versions)
+for details on how to build Flink for a specific Scala version.
+
+**Note:** Because of major breaking changes in Scala 2.12, Flink 1.5 currently builds only for Scala 2.11.
+We aim to add support for Scala 2.12 in the next versions.
+
+
+## Hadoop Dependencies
+
+**General rule: It should never be necessary to add Hadoop dependencies directly to your application.**
+*(The only exception being when using existing Hadoop input-/output formats with Flink's Hadoop compatibility wrappers)*
+
+If you want to use Flink with Hadoop, you need to have a Flink setup that includes the Hadoop dependencies, rather than
+adding Hadoop as an application dependency. Please refer to the [Hadoop Setup Guide]({{ site.baseurl }}/ops/deployment/hadoop.html)
+for details.
+
+There are two main reasons for that design:
+
+ - Some Hadoop interaction happens in Flink's core, possibly before the user application is started, for example
+ setting up HDFS for checkpoints, authenticating via Hadoop's Kerberos tokens, or deployment on YARN.
+
+ - Flink's inverted classloading approach hides many transitive dependencies from the core dependencies. That applies not only
+ to Flink's own core dependencies, but also to Hadoop's dependencies when present in the setup.
+ That way, applications can use different versions of the same dependencies without running into dependency conflicts (and
+ trust us, that's a big deal, because Hadoops dependency tree is huge.)
+
+If you need Hadoop dependencies during testing or development inside the IDE (for example for HDFS access), please configure
+these dependencies similar to the scope of the dependencies to *test* or to *provided*.
+
+
+## Appendix: Template for bulding a Jar with Dependencies
+
+To build an application JAR that contains all dependencies required for declared connectors and libraries,
+you can use the following shade plugin definition:
+
+{% highlight xml %}
+<build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.0.0</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <exclude>com.google.code.findbugs:jsr305</exclude>
+ <exclude>org.slf4j:*</exclude>
+ <exclude>log4j:*</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <!-- Do not copy the signatures in the META-INF folder.
+ Otherwise, this might cause SecurityExceptions when using the JAR. -->
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>my.prorgams.main.clazz</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+</build>
+{% endhighlight %}
+
+{% top %}
+