You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2017/07/24 09:22:05 UTC

[1/4] flink git commit: [FLINK-6539] Add automated batch WordCount end-to-end test

Repository: flink
Updated Branches:
  refs/heads/master 656ac831c -> bb118104b


[FLINK-6539] Add automated batch WordCount end-to-end test


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4f1c764e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4f1c764e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4f1c764e

Branch: refs/heads/master
Commit: 4f1c764e969d8c87db83cb1f8687bb52ec211dbc
Parents: 656ac83
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Tue May 9 13:39:27 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:17:44 2017 +0200

----------------------------------------------------------------------
 pom.xml                                         |   1 +
 test-infra/end-to-end-test/common.sh            | 118 +++++++++++++++++++
 test-infra/end-to-end-test/test-data/words      |   1 +
 .../end-to-end-test/test_batch_wordcount.sh     |  41 +++++++
 4 files changed, 161 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2f14765..8c344b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -974,6 +974,7 @@ under the License.
 						<exclude>flink-connectors/flink-avro/src/test/resources/avro/*.avsc</exclude>
 						<exclude>out/test/flink-avro/avro/user.avsc</exclude>
 						<exclude>flink-libraries/flink-table/src/test/scala/resources/*.out</exclude>
+						<exclude>test-infra/end-to-end-test/test-data/*</exclude>
 
 						<!-- snapshots -->
 						<exclude>**/src/test/resources/*-snapshot</exclude>

http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/test-infra/end-to-end-test/common.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/common.sh b/test-infra/end-to-end-test/common.sh
new file mode 100644
index 0000000..00e4e75
--- /dev/null
+++ b/test-infra/end-to-end-test/common.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+set -e
+set -o pipefail
+
+export FLINK_DIR="$1"
+export CLUSTER_MODE="$2"
+
+export PASS=1
+
+echo "Flink dist directory: $FLINK_DIR"
+
+# used to randomize created directories
+export TEST_DATA_DIR=$TEST_INFRA_DIR/temp-test-directory-$(date +%S%N)
+echo "TEST_DATA_DIR: $TEST_DATA_DIR"
+
+function start_cluster {
+  if [[ "$CLUSTER_MODE" == "local" ]]; then
+    $FLINK_DIR/bin/start-local.sh
+  elif [[ "$CLUSTER_MODE" == "cluster" ]]; then
+    $FLINK_DIR/bin/start-cluster.sh
+  else
+    echo "Unrecognized cluster mode: $CLUSTER_MODE"
+    exit
+  fi
+
+  # wait for TaskManager to come up
+  # wait roughly 10 seconds
+  for i in {1..10}; do
+    # without the || true this would exit our script if the JobManager is not yet up
+    QUERY_RESULT=$(curl "http://localhost:8081/taskmanagers" || true)
+
+    if [[ "$QUERY_RESULT" == "" ]]; then
+     echo "JobManager is not yet up"
+    elif [[ "$QUERY_RESULT" != "{\"taskmanagers\":[]}" ]]; then
+      break
+    fi
+
+    echo "Waiting for cluster to come up..."
+    sleep 1
+  done
+}
+
+function stop_cluster {
+  if [[ "$CLUSTER_MODE" == "local" ]]; then
+    $FLINK_DIR/bin/stop-local.sh
+  elif [[ "$CLUSTER_MODE" == "cluster" ]]; then
+    $FLINK_DIR/bin/stop-cluster.sh
+  fi
+
+  if grep -riq "error" $FLINK_DIR/log; then
+    echo "Found error in log files."
+    PASS=""
+  fi
+  if grep -riq "exception" $FLINK_DIR/log; then
+    echo "Found exception in log files."
+    PASS=""
+  fi
+
+  for f in `ls $FLINK_DIR/log/*.out`
+  do
+    if [[ -s $f ]]; then
+      echo "Found non-empty file $f"
+      PASS=""
+    fi
+  done
+
+  rm $FLINK_DIR/log/*
+}
+
+function check_result_hash {
+  local name=$1
+  local outfile_prefix=$2
+  local expected=$3
+
+  local actual=$(LC_ALL=C sort $outfile_prefix* | md5sum | awk '{print $1}' \
+    || LC_ALL=C sort $outfile_prefix* | md5 -q) || exit 2  # OSX
+  if [[ "$actual" != "$expected" ]]
+  then
+    echo "FAIL $name: Output hash mismatch.  Got $actual, expected $expected."
+    PASS=""
+    echo "head hexdump of actual:"
+    head $outfile_prefix* | hexdump -c
+  else
+    echo "pass $name"
+    # Output files are left behind in /tmp
+  fi
+}
+
+function check_all_pass {
+  if [[ ! "$PASS" ]]
+  then
+    echo "One or more tests FAILED."
+    exit 1
+  fi
+  echo "All tests PASS"
+}
+
+function clean_data_dir {
+  rm -r $TEST_DATA_DIR
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/test-infra/end-to-end-test/test-data/words
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/test-data/words b/test-infra/end-to-end-test/test-data/words
new file mode 100644
index 0000000..0213510
--- /dev/null
+++ b/test-infra/end-to-end-test/test-data/words
@@ -0,0 +1 @@
+Hello World how are you, my dear dear world

http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/test-infra/end-to-end-test/test_batch_wordcount.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/test_batch_wordcount.sh b/test-infra/end-to-end-test/test_batch_wordcount.sh
new file mode 100755
index 0000000..dfde5c6
--- /dev/null
+++ b/test-infra/end-to-end-test/test_batch_wordcount.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+set -e
+set -o pipefail
+
+# Convert relative path to absolute path
+TEST_ROOT=`pwd`
+TEST_INFRA_DIR="$0"
+TEST_INFRA_DIR=`dirname "$TEST_INFRA_DIR"`
+cd $TEST_INFRA_DIR
+TEST_INFRA_DIR=`pwd`
+cd $TEST_ROOT
+
+. "$TEST_INFRA_DIR"/common.sh
+
+start_cluster
+
+$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $TEST_INFRA_DIR/test-data/words --output $TEST_DATA_DIR/out/wc_out
+check_result_hash "WordCount" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5"
+
+stop_cluster
+clean_data_dir
+check_all_pass
\ No newline at end of file


[2/4] flink git commit: [FLINK-6539] Combine ReadFromKafka/WriteIntoKafka into one Kafka010Example

Posted by al...@apache.org.
[FLINK-6539] Combine ReadFromKafka/WriteIntoKafka into one Kafka010Example

This also updates the Kafka version we use in the examples module to
0.10.x.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c86f71be
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c86f71be
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c86f71be

Branch: refs/heads/master
Commit: c86f71be9d83290c60a8dd63afe92ef9dcebac9c
Parents: 4f1c764
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Tue May 9 14:46:39 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:18:06 2017 +0200

----------------------------------------------------------------------
 flink-examples/flink-examples-streaming/pom.xml |  8 +-
 .../examples/kafka/Kafka010Example.java         | 90 ++++++++++++++++++++
 .../streaming/examples/kafka/ReadFromKafka.java | 64 --------------
 .../examples/kafka/WriteIntoKafka.java          | 73 ----------------
 .../scala/examples/kafka/Kafka010Example.scala  | 88 +++++++++++++++++++
 .../scala/examples/kafka/ReadFromKafka.scala    | 73 ----------------
 .../scala/examples/kafka/WriteIntoKafka.scala   | 84 ------------------
 7 files changed, 182 insertions(+), 298 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index b023cd5..eba81d3 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -58,7 +58,7 @@ under the License.
 
 		<dependency>
 			<groupId>org.apache.flink</groupId>
-			<artifactId>flink-connector-kafka-0.8_${scala.binary.version}</artifactId>
+			<artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
 			<version>${project.version}</version>
 		</dependency>
 
@@ -488,7 +488,7 @@ under the License.
 				<artifactId>maven-shade-plugin</artifactId>
 				<executions>
 					<execution>
-						<id>fat-jar-kafka-example</id>
+						<id>fat-jar-kafka-010-example</id>
 						<phase>package</phase>
 						<goals>
 							<goal>shade</goal>
@@ -499,10 +499,10 @@ under the License.
 							<createDependencyReducedPom>false</createDependencyReducedPom>
 							<transformers>
 								<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-									<mainClass>org.apache.flink.streaming.examples.kafka.ReadFromKafka</mainClass>
+									<mainClass>org.apache.flink.streaming.examples.kafka.Kafka010Example</mainClass>
 								</transformer>
 							</transformers>
-							<finalName>Kafka</finalName>
+							<finalName>Kafka010Example</finalName>
 							<!-- <outputFile>Kafka.jar</outputFile> -->
 							<filters>
 								<filter>

http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java
new file mode 100644
index 0000000..b5abbc5
--- /dev/null
+++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.kafka;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
+import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010;
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+
+
+/**
+ * An example that shows how to read from and write to Kafka. This will read String messages
+ * from the input topic, prefix them by a configured prefix and output to the output topic.
+ *
+ * <p>Example usage:
+ * 	--input-topic test-input --output-topic test-output --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer
+ */
+public class Kafka010Example {
+
+	public static void main(String[] args) throws Exception {
+		// parse input arguments
+		final ParameterTool parameterTool = ParameterTool.fromArgs(args);
+
+		if (parameterTool.getNumberOfParameters() < 5) {
+			System.out.println("Missing parameters!\n" +
+					"Usage: Kafka --input-topic <topic> --output-topic <topic> " +
+					"--bootstrap.servers <kafka brokers> " +
+					"--zookeeper.connect <zk quorum> --group.id <some id> [--prefix <prefix>]");
+			return;
+		}
+
+		String prefix = parameterTool.get("prefix", "PREFIX:");
+
+		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+		env.getConfig().disableSysoutLogging();
+		env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
+		env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
+		env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
+
+		// make parameters available in the web interface
+		env.getConfig().setGlobalJobParameters(parameterTool);
+
+		DataStream<String> input = env
+				.addSource(new FlinkKafkaConsumer010<>(
+						parameterTool.getRequired("input-topic"),
+						new SimpleStringSchema(),
+						parameterTool.getProperties()))
+				.map(new PrefixingMapper(prefix));
+
+		input.addSink(
+				new FlinkKafkaProducer010<>(
+						parameterTool.getRequired("output-topic"),
+						new SimpleStringSchema(),
+						parameterTool.getProperties()));
+
+		env.execute("Kafka 0.10 Example");
+	}
+
+	private static class PrefixingMapper implements MapFunction<String, String> {
+		private final String prefix;
+
+		public PrefixingMapper(String prefix) {
+			this.prefix = prefix;
+		}
+
+		@Override
+		public String map(String value) throws Exception {
+			return prefix + value;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java
deleted file mode 100644
index f9cf42b..0000000
--- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.kafka;
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-
-
-/**
- * Read Strings from Kafka and print them to standard out.
- * Note: On a cluster, DataStream.print() will print to the TaskManager's .out file!
- *
- * <p>Please pass the following arguments to run the example:
- * 	--topic test --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer
- */
-public class ReadFromKafka {
-
-	public static void main(String[] args) throws Exception {
-		// parse input arguments
-		final ParameterTool parameterTool = ParameterTool.fromArgs(args);
-
-		if (parameterTool.getNumberOfParameters() < 4) {
-			System.out.println("Missing parameters!\nUsage: Kafka --topic <topic> " +
-					"--bootstrap.servers <kafka brokers> --zookeeper.connect <zk quorum> --group.id <some id>");
-			return;
-		}
-
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-		env.getConfig().disableSysoutLogging();
-		env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
-		env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
-		env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
-
-		DataStream<String> messageStream = env
-				.addSource(new FlinkKafkaConsumer08<>(
-						parameterTool.getRequired("topic"),
-						new SimpleStringSchema(),
-						parameterTool.getProperties()));
-
-		// write kafka stream to standard out.
-		messageStream.print();
-
-		env.execute("Read from Kafka example");
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java
deleted file mode 100644
index f9b4656..0000000
--- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.kafka;
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer08;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-
-/**
- * Generate a String every 500 ms and write it into a Kafka topic.
- *
- * <p>Please pass the following arguments to run the example:
- * 	--topic test --bootstrap.servers localhost:9092
- */
-public class WriteIntoKafka {
-
-	public static void main(String[] args) throws Exception {
-		ParameterTool parameterTool = ParameterTool.fromArgs(args);
-		if (parameterTool.getNumberOfParameters() < 2) {
-			System.out.println("Missing parameters!");
-			System.out.println("Usage: Kafka --topic <topic> --bootstrap.servers <kafka brokers>");
-			return;
-		}
-
-		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-		env.getConfig().disableSysoutLogging();
-		env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
-
-		// very simple data generator
-		DataStream<String> messageStream = env.addSource(new SourceFunction<String>() {
-			private static final long serialVersionUID = 6369260445318862378L;
-			public boolean running = true;
-
-			@Override
-			public void run(SourceContext<String> ctx) throws Exception {
-				long i = 0;
-				while (this.running) {
-					ctx.collect("Element - " + i++);
-					Thread.sleep(500);
-				}
-			}
-
-			@Override
-			public void cancel() {
-				running = false;
-			}
-		});
-
-		// write data into Kafka
-		messageStream.addSink(new FlinkKafkaProducer08<>(parameterTool.getRequired("topic"), new SimpleStringSchema(), parameterTool.getProperties()));
-
-		env.execute("Write into Kafka example");
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala
new file mode 100644
index 0000000..2a52811
--- /dev/null
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.kafka
+
+import org.apache.flink.api.common.restartstrategy.RestartStrategies
+import org.apache.flink.api.java.utils.ParameterTool
+import org.apache.flink.streaming.api.scala._
+import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer010, FlinkKafkaProducer010}
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema
+
+/**
+ * An example that shows how to read from and write to Kafka. This will read String messages
+ * from the input topic, prefix them by a configured prefix and output to the output topic.
+ *
+ * Please pass the following arguments to run the example:
+ * {{{
+ * --input-topic test-input
+ * --output-topic test-output
+ * --bootstrap.servers localhost:9092
+ * --zookeeper.connect localhost:2181
+ * --group.id myconsumer
+ * }}}
+ */
+object Kafka010Example {
+
+  def main(args: Array[String]): Unit = {
+
+    // parse input arguments
+    val params = ParameterTool.fromArgs(args)
+
+    if (params.getNumberOfParameters < 4) {
+      println("Missing parameters!\n"
+        + "Usage: Kafka --input-topic <topic> --output-topic <topic> "
+        + "--bootstrap.servers <kafka brokers> "
+        + "--zookeeper.connect <zk quorum> --group.id <some id> [--prefix <prefix>]")
+      return
+    }
+
+    val prefix = params.get("prefix", "PREFIX:")
+
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.getConfig.disableSysoutLogging
+    env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000))
+    // create a checkpoint every 5 seconds
+    env.enableCheckpointing(5000)
+    // make parameters available in the web interface
+    env.getConfig.setGlobalJobParameters(params)
+
+    // create a Kafka streaming source consumer for Kafka 0.10.x
+    val kafkaConsumer = new FlinkKafkaConsumer010(
+      params.getRequired("input-topic"),
+      new SimpleStringSchema,
+      params.getProperties)
+
+    val messageStream = env
+      .addSource(kafkaConsumer)
+      .map(in => prefix + in)
+
+    // create a Kafka producer for Kafka 0.10.x
+    val kafkaProducer = new FlinkKafkaProducer010(
+      params.getRequired("output-topic"),
+      new SimpleStringSchema,
+      params.getProperties)
+
+    // write data into Kafka
+    messageStream.addSink(kafkaProducer)
+
+    env.execute("Kafka 0.10 Example")
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala
deleted file mode 100644
index 3127ab7..0000000
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.kafka
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies
-import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema
-
-/**
- * Read Strings from Kafka and print them to standard out.
- * Note: On a cluster, DataStream.print() will print to the TaskManager's .out file!
- *
- * Please pass the following arguments to run the example:
- * {{{
- * --topic test
- * --bootstrap.servers localhost:9092
- * --zookeeper.connect localhost:2181
- * --group.id myconsumer
- * }}}
- */
-object ReadFromKafka {
-
-  def main(args: Array[String]): Unit = {
-
-    // parse input arguments
-    val params = ParameterTool.fromArgs(args)
-
-    if (params.getNumberOfParameters < 4) {
-      println("Missing parameters!\nUsage: Kafka --topic <topic> " +
-        "--bootstrap.servers <kafka brokers> --zookeeper.connect <zk quorum> --group.id <some id>")
-      return
-    }
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.getConfig.disableSysoutLogging
-    env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000))
-    // create a checkpoint every 5 seconds
-    env.enableCheckpointing(5000)
-    // make parameters available in the web interface
-    env.getConfig.setGlobalJobParameters(params)
-
-    // create a Kafka streaming source consumer for Kafka 0.8.x
-    val kafkaConsumer = new FlinkKafkaConsumer08(
-      params.getRequired("topic"),
-      new SimpleStringSchema,
-      params.getProperties)
-    val messageStream = env.addSource(kafkaConsumer)
-
-    // write kafka stream to standard out.
-    messageStream.print()
-
-    env.execute("Read from Kafka example")
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala
deleted file mode 100644
index e34083a..0000000
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.kafka
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies
-import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.streaming.api.functions.source.SourceFunction
-import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer08
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema
-
-/**
- * Generate a String every 500 ms and write it into a Kafka topic
- *
- * Please pass the following arguments to run the example:
- * {{{
- * --topic test
- * --bootstrap.servers
- * localhost:9092
- * }}}
- */
-object WriteIntoKafka {
-
-  def main(args: Array[String]): Unit = {
-
-    // parse input arguments
-    val params = ParameterTool.fromArgs(args)
-
-    if (params.getNumberOfParameters < 2) {
-      println("Missing parameters!")
-      println("Usage: Kafka --topic <topic> --bootstrap.servers <kafka brokers>")
-      return
-    }
-
-    val env = StreamExecutionEnvironment.getExecutionEnvironment
-    env.getConfig.disableSysoutLogging
-    env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000))
-
-    // very simple data generator
-    val messageStream: DataStream[String] = env.addSource(new SourceFunction[String]() {
-      var running = true
-
-      override def run(ctx: SourceContext[String]): Unit = {
-        var i = 0L
-        while (this.running) {
-          ctx.collect(s"Element - ${i}")
-          i += 1
-          Thread.sleep(500)
-        }
-      }
-
-      override def cancel(): Unit = running = false
-    })
-
-    // create a Kafka producer for Kafka 0.8.x
-    val kafkaProducer = new FlinkKafkaProducer08(
-      params.getRequired("topic"),
-      new SimpleStringSchema,
-      params.getProperties)
-
-    // write data into Kafka
-    messageStream.addSink(kafkaProducer)
-
-    env.execute("Write into Kafka example")
-  }
-
-}


[3/4] flink git commit: [FLINK-6539] Add automated Kafka end-to-end test

Posted by al...@apache.org.
[FLINK-6539] Add automated Kafka end-to-end test


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1a5930cb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1a5930cb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1a5930cb

Branch: refs/heads/master
Commit: 1a5930cb7344cef5c4ddffeb4104583846945a16
Parents: c86f71b
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Thu May 11 12:08:12 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:18:06 2017 +0200

----------------------------------------------------------------------
 test-infra/end-to-end-test/common.sh            |  11 +-
 .../end-to-end-test/test_streaming_kafka010.sh  | 108 +++++++++++++++++++
 2 files changed, 115 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/1a5930cb/test-infra/end-to-end-test/common.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/common.sh b/test-infra/end-to-end-test/common.sh
index 00e4e75..e05f70f 100644
--- a/test-infra/end-to-end-test/common.sh
+++ b/test-infra/end-to-end-test/common.sh
@@ -65,12 +65,14 @@ function stop_cluster {
     $FLINK_DIR/bin/stop-cluster.sh
   fi
 
-  if grep -riq "error" $FLINK_DIR/log; then
-    echo "Found error in log files."
+  if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log | grep -v "RetriableCommitFailedException" | grep -iq "error"; then
+    echo "Found error in log files:"
+    cat $FLINK_DIR/log/*
     PASS=""
   fi
-  if grep -riq "exception" $FLINK_DIR/log; then
-    echo "Found exception in log files."
+  if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log | grep -v "RetriableCommitFailedException" | grep -iq "exception"; then
+    echo "Found exception in log files:"
+    cat $FLINK_DIR/log/*
     PASS=""
   fi
 
@@ -78,6 +80,7 @@ function stop_cluster {
   do
     if [[ -s $f ]]; then
       echo "Found non-empty file $f"
+      cat $f
       PASS=""
     fi
   done

http://git-wip-us.apache.org/repos/asf/flink/blob/1a5930cb/test-infra/end-to-end-test/test_streaming_kafka010.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/test_streaming_kafka010.sh b/test-infra/end-to-end-test/test_streaming_kafka010.sh
new file mode 100755
index 0000000..638c34b
--- /dev/null
+++ b/test-infra/end-to-end-test/test_streaming_kafka010.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+set -e
+set -o pipefail
+
+# Convert relative path to absolute path
+TEST_ROOT=`pwd`
+TEST_INFRA_DIR="$0"
+TEST_INFRA_DIR=`dirname "$TEST_INFRA_DIR"`
+cd $TEST_INFRA_DIR
+TEST_INFRA_DIR=`pwd`
+cd $TEST_ROOT
+
+. "$TEST_INFRA_DIR"/common.sh
+
+
+start_cluster
+
+# get Kafka 0.10.0
+mkdir -p $TEST_DATA_DIR
+if [ -z "$3" ]; then
+  # need to download Kafka because no Kafka was specified on the invocation
+  KAFKA_URL="http://mirror.netcologne.de/apache.org/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz"
+  echo "Downloading Kafka from $KAFKA_URL"
+  curl "$KAFKA_URL" > $TEST_DATA_DIR/kafka.tgz
+else
+  echo "Using specified Kafka from $3"
+  cp $3 $TEST_DATA_DIR/kafka.tgz
+fi
+
+tar xzf $TEST_DATA_DIR/kafka.tgz -C $TEST_DATA_DIR/
+KAFKA_DIR=$TEST_DATA_DIR/kafka_2.11-0.10.2.0
+
+# fix kafka config
+sed -i -e "s+^\(dataDir\s*=\s*\).*$+\1$TEST_DATA_DIR/zookeeper+" $KAFKA_DIR/config/zookeeper.properties
+sed -i -e "s+^\(log\.dirs\s*=\s*\).*$+\1$TEST_DATA_DIR/kafka+" $KAFKA_DIR/config/server.properties
+$KAFKA_DIR/bin/zookeeper-server-start.sh -daemon $KAFKA_DIR/config/zookeeper.properties
+$KAFKA_DIR/bin/kafka-server-start.sh -daemon $KAFKA_DIR/config/server.properties
+
+# zookeeper outputs the "Node does not exist" bit to stderr
+while [[ $($KAFKA_DIR/bin/zookeeper-shell.sh localhost:2181 get /brokers/ids/0 2>&1) =~ .*Node\ does\ not\ exist.* ]]; do
+  echo "Waiting for broker..."
+  sleep 1
+done
+
+# create the required topics
+$KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-input
+$KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-output
+
+# run the Flink job (detached mode)
+$FLINK_DIR/bin/flink run -d build-target/examples/streaming/Kafka010Example.jar \
+  --input-topic test-input --output-topic test-output \
+  --prefix=PREFIX \
+  --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer --auto.offset.reset earliest
+
+# send some data to Kafka
+echo -e "hello\nwhats\nup" | $KAFKA_DIR/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test-input
+
+# wait at most (roughly) 60 seconds until the results are there
+for i in {1..60}; do
+  DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --timeout-ms 0 2> /dev/null)
+
+  # make sure we have actual newlines in the string, not "\n"
+  EXPECTED=$(printf "PREFIX:hello\nPREFIX:whats\nPREFIX:up")
+
+  if [[ "$DATA_FROM_KAFKA" == "$EXPECTED" ]]; then
+    break
+  fi
+
+  echo "Waiting for results from Kafka..."
+  sleep 1
+done
+
+# verify again to set the PASS variable
+DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --timeout-ms 0 2> /dev/null)
+
+# make sure we have actual newlines in the string, not "\n"
+EXPECTED=$(printf "PREFIX:hello\nPREFIX:whats\nPREFIX:up")
+if [[ "$DATA_FROM_KAFKA" != "$EXPECTED" ]]; then
+  echo "Output from Flink program does not match expected output."
+  echo -e "EXPECTED: --$EXPECTED--"
+  echo -e "ACTUAL: --$DATA_FROM_KAFKA--"
+  PASS=""
+fi
+
+$KAFKA_DIR/bin/kafka-server-stop.sh
+$KAFKA_DIR/bin/zookeeper-server-stop.sh
+
+stop_cluster
+clean_data_dir
+check_all_pass


[4/4] flink git commit: [FLINK-6539] Run end-to-end tests on travis

Posted by al...@apache.org.
[FLINK-6539] Run end-to-end tests on travis


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bb118104
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bb118104
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bb118104

Branch: refs/heads/master
Commit: bb118104bc71e00e097b799d1cadb7f95736f69b
Parents: 1a5930c
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Thu May 11 12:17:47 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:20:04 2017 +0200

----------------------------------------------------------------------
 tools/travis_mvn_watchdog.sh | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/bb118104/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 1c1172e..c2a51a5 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -274,7 +274,7 @@ check_shaded_artifacts() {
 		echo "=============================================================================="
 		return 1
 	fi
-	 
+
 	GUAVA=`cat allClasses | grep '^com/google/common' | wc -l`
 	if [ $GUAVA != "0" ]; then
 		echo "=============================================================================="
@@ -399,5 +399,17 @@ upload_artifacts_s3
 # we are going back to
 cd ../../
 
+# only run end-to-end tests in misc because we only have flink-dist here
+case $TEST in
+	(misc)
+        echo "Running automated end-to-end tests"
+
+        test-infra/end-to-end-test/test_batch_wordcount.sh build-target cluster
+        EXIT_CODE=$(($EXIT_CODE+$?))
+        test-infra/end-to-end-test/test_streaming_kafka010.sh build-target cluster
+        EXIT_CODE=$(($EXIT_CODE+$?))
+	;;
+esac
+
 # Exit code for Travis build success/failure
 exit $EXIT_CODE