You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2017/07/24 09:22:05 UTC
[1/4] flink git commit: [FLINK-6539] Add automated batch WordCount
end-to-end test
Repository: flink
Updated Branches:
refs/heads/master 656ac831c -> bb118104b
[FLINK-6539] Add automated batch WordCount end-to-end test
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/4f1c764e
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/4f1c764e
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/4f1c764e
Branch: refs/heads/master
Commit: 4f1c764e969d8c87db83cb1f8687bb52ec211dbc
Parents: 656ac83
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Tue May 9 13:39:27 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:17:44 2017 +0200
----------------------------------------------------------------------
pom.xml | 1 +
test-infra/end-to-end-test/common.sh | 118 +++++++++++++++++++
test-infra/end-to-end-test/test-data/words | 1 +
.../end-to-end-test/test_batch_wordcount.sh | 41 +++++++
4 files changed, 161 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2f14765..8c344b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -974,6 +974,7 @@ under the License.
<exclude>flink-connectors/flink-avro/src/test/resources/avro/*.avsc</exclude>
<exclude>out/test/flink-avro/avro/user.avsc</exclude>
<exclude>flink-libraries/flink-table/src/test/scala/resources/*.out</exclude>
+ <exclude>test-infra/end-to-end-test/test-data/*</exclude>
<!-- snapshots -->
<exclude>**/src/test/resources/*-snapshot</exclude>
http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/test-infra/end-to-end-test/common.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/common.sh b/test-infra/end-to-end-test/common.sh
new file mode 100644
index 0000000..00e4e75
--- /dev/null
+++ b/test-infra/end-to-end-test/common.sh
@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+set -e
+set -o pipefail
+
+export FLINK_DIR="$1"
+export CLUSTER_MODE="$2"
+
+export PASS=1
+
+echo "Flink dist directory: $FLINK_DIR"
+
+# used to randomize created directories
+export TEST_DATA_DIR=$TEST_INFRA_DIR/temp-test-directory-$(date +%S%N)
+echo "TEST_DATA_DIR: $TEST_DATA_DIR"
+
+function start_cluster {
+ if [[ "$CLUSTER_MODE" == "local" ]]; then
+ $FLINK_DIR/bin/start-local.sh
+ elif [[ "$CLUSTER_MODE" == "cluster" ]]; then
+ $FLINK_DIR/bin/start-cluster.sh
+ else
+ echo "Unrecognized cluster mode: $CLUSTER_MODE"
+ exit
+ fi
+
+ # wait for TaskManager to come up
+ # wait roughly 10 seconds
+ for i in {1..10}; do
+ # without the || true this would exit our script if the JobManager is not yet up
+ QUERY_RESULT=$(curl "http://localhost:8081/taskmanagers" || true)
+
+ if [[ "$QUERY_RESULT" == "" ]]; then
+ echo "JobManager is not yet up"
+ elif [[ "$QUERY_RESULT" != "{\"taskmanagers\":[]}" ]]; then
+ break
+ fi
+
+ echo "Waiting for cluster to come up..."
+ sleep 1
+ done
+}
+
+function stop_cluster {
+ if [[ "$CLUSTER_MODE" == "local" ]]; then
+ $FLINK_DIR/bin/stop-local.sh
+ elif [[ "$CLUSTER_MODE" == "cluster" ]]; then
+ $FLINK_DIR/bin/stop-cluster.sh
+ fi
+
+ if grep -riq "error" $FLINK_DIR/log; then
+ echo "Found error in log files."
+ PASS=""
+ fi
+ if grep -riq "exception" $FLINK_DIR/log; then
+ echo "Found exception in log files."
+ PASS=""
+ fi
+
+ for f in `ls $FLINK_DIR/log/*.out`
+ do
+ if [[ -s $f ]]; then
+ echo "Found non-empty file $f"
+ PASS=""
+ fi
+ done
+
+ rm $FLINK_DIR/log/*
+}
+
+function check_result_hash {
+ local name=$1
+ local outfile_prefix=$2
+ local expected=$3
+
+ local actual=$(LC_ALL=C sort $outfile_prefix* | md5sum | awk '{print $1}' \
+ || LC_ALL=C sort $outfile_prefix* | md5 -q) || exit 2 # OSX
+ if [[ "$actual" != "$expected" ]]
+ then
+ echo "FAIL $name: Output hash mismatch. Got $actual, expected $expected."
+ PASS=""
+ echo "head hexdump of actual:"
+ head $outfile_prefix* | hexdump -c
+ else
+ echo "pass $name"
+ # Output files are left behind in /tmp
+ fi
+}
+
+function check_all_pass {
+ if [[ ! "$PASS" ]]
+ then
+ echo "One or more tests FAILED."
+ exit 1
+ fi
+ echo "All tests PASS"
+}
+
+function clean_data_dir {
+ rm -r $TEST_DATA_DIR
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/test-infra/end-to-end-test/test-data/words
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/test-data/words b/test-infra/end-to-end-test/test-data/words
new file mode 100644
index 0000000..0213510
--- /dev/null
+++ b/test-infra/end-to-end-test/test-data/words
@@ -0,0 +1 @@
+Hello World how are you, my dear dear world
http://git-wip-us.apache.org/repos/asf/flink/blob/4f1c764e/test-infra/end-to-end-test/test_batch_wordcount.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/test_batch_wordcount.sh b/test-infra/end-to-end-test/test_batch_wordcount.sh
new file mode 100755
index 0000000..dfde5c6
--- /dev/null
+++ b/test-infra/end-to-end-test/test_batch_wordcount.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+set -e
+set -o pipefail
+
+# Convert relative path to absolute path
+TEST_ROOT=`pwd`
+TEST_INFRA_DIR="$0"
+TEST_INFRA_DIR=`dirname "$TEST_INFRA_DIR"`
+cd $TEST_INFRA_DIR
+TEST_INFRA_DIR=`pwd`
+cd $TEST_ROOT
+
+. "$TEST_INFRA_DIR"/common.sh
+
+start_cluster
+
+$FLINK_DIR/bin/flink run -p 1 $FLINK_DIR/examples/batch/WordCount.jar --input $TEST_INFRA_DIR/test-data/words --output $TEST_DATA_DIR/out/wc_out
+check_result_hash "WordCount" $TEST_DATA_DIR/out/wc_out "72a690412be8928ba239c2da967328a5"
+
+stop_cluster
+clean_data_dir
+check_all_pass
\ No newline at end of file
[2/4] flink git commit: [FLINK-6539] Combine
ReadFromKafka/WriteIntoKafka into one Kafka010Example
Posted by al...@apache.org.
[FLINK-6539] Combine ReadFromKafka/WriteIntoKafka into one Kafka010Example
This also updates the Kafka version we use in the examples module to
0.10.x.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c86f71be
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c86f71be
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c86f71be
Branch: refs/heads/master
Commit: c86f71be9d83290c60a8dd63afe92ef9dcebac9c
Parents: 4f1c764
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Tue May 9 14:46:39 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:18:06 2017 +0200
----------------------------------------------------------------------
flink-examples/flink-examples-streaming/pom.xml | 8 +-
.../examples/kafka/Kafka010Example.java | 90 ++++++++++++++++++++
.../streaming/examples/kafka/ReadFromKafka.java | 64 --------------
.../examples/kafka/WriteIntoKafka.java | 73 ----------------
.../scala/examples/kafka/Kafka010Example.scala | 88 +++++++++++++++++++
.../scala/examples/kafka/ReadFromKafka.scala | 73 ----------------
.../scala/examples/kafka/WriteIntoKafka.scala | 84 ------------------
7 files changed, 182 insertions(+), 298 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index b023cd5..eba81d3 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -58,7 +58,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-kafka-0.8_${scala.binary.version}</artifactId>
+ <artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
@@ -488,7 +488,7 @@ under the License.
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
- <id>fat-jar-kafka-example</id>
+ <id>fat-jar-kafka-010-example</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
@@ -499,10 +499,10 @@ under the License.
<createDependencyReducedPom>false</createDependencyReducedPom>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.apache.flink.streaming.examples.kafka.ReadFromKafka</mainClass>
+ <mainClass>org.apache.flink.streaming.examples.kafka.Kafka010Example</mainClass>
</transformer>
</transformers>
- <finalName>Kafka</finalName>
+ <finalName>Kafka010Example</finalName>
<!-- <outputFile>Kafka.jar</outputFile> -->
<filters>
<filter>
http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java
new file mode 100644
index 0000000..b5abbc5
--- /dev/null
+++ b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/Kafka010Example.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.examples.kafka;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
+import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer010;
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
+
+
+/**
+ * An example that shows how to read from and write to Kafka. This will read String messages
+ * from the input topic, prefix them by a configured prefix and output to the output topic.
+ *
+ * <p>Example usage:
+ * --input-topic test-input --output-topic test-output --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer
+ */
+public class Kafka010Example {
+
+ public static void main(String[] args) throws Exception {
+ // parse input arguments
+ final ParameterTool parameterTool = ParameterTool.fromArgs(args);
+
+ if (parameterTool.getNumberOfParameters() < 5) {
+ System.out.println("Missing parameters!\n" +
+ "Usage: Kafka --input-topic <topic> --output-topic <topic> " +
+ "--bootstrap.servers <kafka brokers> " +
+ "--zookeeper.connect <zk quorum> --group.id <some id> [--prefix <prefix>]");
+ return;
+ }
+
+ String prefix = parameterTool.get("prefix", "PREFIX:");
+
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.getConfig().disableSysoutLogging();
+ env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
+ env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
+ env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
+
+ // make parameters available in the web interface
+ env.getConfig().setGlobalJobParameters(parameterTool);
+
+ DataStream<String> input = env
+ .addSource(new FlinkKafkaConsumer010<>(
+ parameterTool.getRequired("input-topic"),
+ new SimpleStringSchema(),
+ parameterTool.getProperties()))
+ .map(new PrefixingMapper(prefix));
+
+ input.addSink(
+ new FlinkKafkaProducer010<>(
+ parameterTool.getRequired("output-topic"),
+ new SimpleStringSchema(),
+ parameterTool.getProperties()));
+
+ env.execute("Kafka 0.10 Example");
+ }
+
+ private static class PrefixingMapper implements MapFunction<String, String> {
+ private final String prefix;
+
+ public PrefixingMapper(String prefix) {
+ this.prefix = prefix;
+ }
+
+ @Override
+ public String map(String value) throws Exception {
+ return prefix + value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java
deleted file mode 100644
index f9cf42b..0000000
--- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/ReadFromKafka.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.kafka;
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-
-
-/**
- * Read Strings from Kafka and print them to standard out.
- * Note: On a cluster, DataStream.print() will print to the TaskManager's .out file!
- *
- * <p>Please pass the following arguments to run the example:
- * --topic test --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer
- */
-public class ReadFromKafka {
-
- public static void main(String[] args) throws Exception {
- // parse input arguments
- final ParameterTool parameterTool = ParameterTool.fromArgs(args);
-
- if (parameterTool.getNumberOfParameters() < 4) {
- System.out.println("Missing parameters!\nUsage: Kafka --topic <topic> " +
- "--bootstrap.servers <kafka brokers> --zookeeper.connect <zk quorum> --group.id <some id>");
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().disableSysoutLogging();
- env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
- env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
- env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
-
- DataStream<String> messageStream = env
- .addSource(new FlinkKafkaConsumer08<>(
- parameterTool.getRequired("topic"),
- new SimpleStringSchema(),
- parameterTool.getProperties()));
-
- // write kafka stream to standard out.
- messageStream.print();
-
- env.execute("Read from Kafka example");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java
deleted file mode 100644
index f9b4656..0000000
--- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/kafka/WriteIntoKafka.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.kafka;
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer08;
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema;
-
-/**
- * Generate a String every 500 ms and write it into a Kafka topic.
- *
- * <p>Please pass the following arguments to run the example:
- * --topic test --bootstrap.servers localhost:9092
- */
-public class WriteIntoKafka {
-
- public static void main(String[] args) throws Exception {
- ParameterTool parameterTool = ParameterTool.fromArgs(args);
- if (parameterTool.getNumberOfParameters() < 2) {
- System.out.println("Missing parameters!");
- System.out.println("Usage: Kafka --topic <topic> --bootstrap.servers <kafka brokers>");
- return;
- }
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().disableSysoutLogging();
- env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
-
- // very simple data generator
- DataStream<String> messageStream = env.addSource(new SourceFunction<String>() {
- private static final long serialVersionUID = 6369260445318862378L;
- public boolean running = true;
-
- @Override
- public void run(SourceContext<String> ctx) throws Exception {
- long i = 0;
- while (this.running) {
- ctx.collect("Element - " + i++);
- Thread.sleep(500);
- }
- }
-
- @Override
- public void cancel() {
- running = false;
- }
- });
-
- // write data into Kafka
- messageStream.addSink(new FlinkKafkaProducer08<>(parameterTool.getRequired("topic"), new SimpleStringSchema(), parameterTool.getProperties()));
-
- env.execute("Write into Kafka example");
- }
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala
new file mode 100644
index 0000000..2a52811
--- /dev/null
+++ b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/Kafka010Example.scala
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.scala.examples.kafka
+
+import org.apache.flink.api.common.restartstrategy.RestartStrategies
+import org.apache.flink.api.java.utils.ParameterTool
+import org.apache.flink.streaming.api.scala._
+import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer010, FlinkKafkaProducer010}
+import org.apache.flink.streaming.util.serialization.SimpleStringSchema
+
+/**
+ * An example that shows how to read from and write to Kafka. This will read String messages
+ * from the input topic, prefix them by a configured prefix and output to the output topic.
+ *
+ * Please pass the following arguments to run the example:
+ * {{{
+ * --input-topic test-input
+ * --output-topic test-output
+ * --bootstrap.servers localhost:9092
+ * --zookeeper.connect localhost:2181
+ * --group.id myconsumer
+ * }}}
+ */
+object Kafka010Example {
+
+ def main(args: Array[String]): Unit = {
+
+ // parse input arguments
+ val params = ParameterTool.fromArgs(args)
+
+ if (params.getNumberOfParameters < 4) {
+ println("Missing parameters!\n"
+ + "Usage: Kafka --input-topic <topic> --output-topic <topic> "
+ + "--bootstrap.servers <kafka brokers> "
+ + "--zookeeper.connect <zk quorum> --group.id <some id> [--prefix <prefix>]")
+ return
+ }
+
+ val prefix = params.get("prefix", "PREFIX:")
+
+
+ val env = StreamExecutionEnvironment.getExecutionEnvironment
+ env.getConfig.disableSysoutLogging
+ env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000))
+ // create a checkpoint every 5 seconds
+ env.enableCheckpointing(5000)
+ // make parameters available in the web interface
+ env.getConfig.setGlobalJobParameters(params)
+
+ // create a Kafka streaming source consumer for Kafka 0.10.x
+ val kafkaConsumer = new FlinkKafkaConsumer010(
+ params.getRequired("input-topic"),
+ new SimpleStringSchema,
+ params.getProperties)
+
+ val messageStream = env
+ .addSource(kafkaConsumer)
+ .map(in => prefix + in)
+
+ // create a Kafka producer for Kafka 0.10.x
+ val kafkaProducer = new FlinkKafkaProducer010(
+ params.getRequired("output-topic"),
+ new SimpleStringSchema,
+ params.getProperties)
+
+ // write data into Kafka
+ messageStream.addSink(kafkaProducer)
+
+ env.execute("Kafka 0.10 Example")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala
deleted file mode 100644
index 3127ab7..0000000
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/ReadFromKafka.scala
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.kafka
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies
-import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer08
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema
-
-/**
- * Read Strings from Kafka and print them to standard out.
- * Note: On a cluster, DataStream.print() will print to the TaskManager's .out file!
- *
- * Please pass the following arguments to run the example:
- * {{{
- * --topic test
- * --bootstrap.servers localhost:9092
- * --zookeeper.connect localhost:2181
- * --group.id myconsumer
- * }}}
- */
-object ReadFromKafka {
-
- def main(args: Array[String]): Unit = {
-
- // parse input arguments
- val params = ParameterTool.fromArgs(args)
-
- if (params.getNumberOfParameters < 4) {
- println("Missing parameters!\nUsage: Kafka --topic <topic> " +
- "--bootstrap.servers <kafka brokers> --zookeeper.connect <zk quorum> --group.id <some id>")
- return
- }
-
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.getConfig.disableSysoutLogging
- env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000))
- // create a checkpoint every 5 seconds
- env.enableCheckpointing(5000)
- // make parameters available in the web interface
- env.getConfig.setGlobalJobParameters(params)
-
- // create a Kafka streaming source consumer for Kafka 0.8.x
- val kafkaConsumer = new FlinkKafkaConsumer08(
- params.getRequired("topic"),
- new SimpleStringSchema,
- params.getProperties)
- val messageStream = env.addSource(kafkaConsumer)
-
- // write kafka stream to standard out.
- messageStream.print()
-
- env.execute("Read from Kafka example")
- }
-
-}
http://git-wip-us.apache.org/repos/asf/flink/blob/c86f71be/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala
----------------------------------------------------------------------
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala
deleted file mode 100644
index e34083a..0000000
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/kafka/WriteIntoKafka.scala
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.kafka
-
-import org.apache.flink.api.common.restartstrategy.RestartStrategies
-import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.streaming.api.functions.source.SourceFunction
-import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
-import org.apache.flink.streaming.api.scala._
-import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer08
-import org.apache.flink.streaming.util.serialization.SimpleStringSchema
-
-/**
- * Generate a String every 500 ms and write it into a Kafka topic
- *
- * Please pass the following arguments to run the example:
- * {{{
- * --topic test
- * --bootstrap.servers
- * localhost:9092
- * }}}
- */
-object WriteIntoKafka {
-
- def main(args: Array[String]): Unit = {
-
- // parse input arguments
- val params = ParameterTool.fromArgs(args)
-
- if (params.getNumberOfParameters < 2) {
- println("Missing parameters!")
- println("Usage: Kafka --topic <topic> --bootstrap.servers <kafka brokers>")
- return
- }
-
- val env = StreamExecutionEnvironment.getExecutionEnvironment
- env.getConfig.disableSysoutLogging
- env.getConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000))
-
- // very simple data generator
- val messageStream: DataStream[String] = env.addSource(new SourceFunction[String]() {
- var running = true
-
- override def run(ctx: SourceContext[String]): Unit = {
- var i = 0L
- while (this.running) {
- ctx.collect(s"Element - ${i}")
- i += 1
- Thread.sleep(500)
- }
- }
-
- override def cancel(): Unit = running = false
- })
-
- // create a Kafka producer for Kafka 0.8.x
- val kafkaProducer = new FlinkKafkaProducer08(
- params.getRequired("topic"),
- new SimpleStringSchema,
- params.getProperties)
-
- // write data into Kafka
- messageStream.addSink(kafkaProducer)
-
- env.execute("Write into Kafka example")
- }
-
-}
[3/4] flink git commit: [FLINK-6539] Add automated Kafka end-to-end
test
Posted by al...@apache.org.
[FLINK-6539] Add automated Kafka end-to-end test
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1a5930cb
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1a5930cb
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1a5930cb
Branch: refs/heads/master
Commit: 1a5930cb7344cef5c4ddffeb4104583846945a16
Parents: c86f71b
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Thu May 11 12:08:12 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:18:06 2017 +0200
----------------------------------------------------------------------
test-infra/end-to-end-test/common.sh | 11 +-
.../end-to-end-test/test_streaming_kafka010.sh | 108 +++++++++++++++++++
2 files changed, 115 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1a5930cb/test-infra/end-to-end-test/common.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/common.sh b/test-infra/end-to-end-test/common.sh
index 00e4e75..e05f70f 100644
--- a/test-infra/end-to-end-test/common.sh
+++ b/test-infra/end-to-end-test/common.sh
@@ -65,12 +65,14 @@ function stop_cluster {
$FLINK_DIR/bin/stop-cluster.sh
fi
- if grep -riq "error" $FLINK_DIR/log; then
- echo "Found error in log files."
+ if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log | grep -v "RetriableCommitFailedException" | grep -iq "error"; then
+ echo "Found error in log files:"
+ cat $FLINK_DIR/log/*
PASS=""
fi
- if grep -riq "exception" $FLINK_DIR/log; then
- echo "Found exception in log files."
+ if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log | grep -v "RetriableCommitFailedException" | grep -iq "exception"; then
+ echo "Found exception in log files:"
+ cat $FLINK_DIR/log/*
PASS=""
fi
@@ -78,6 +80,7 @@ function stop_cluster {
do
if [[ -s $f ]]; then
echo "Found non-empty file $f"
+ cat $f
PASS=""
fi
done
http://git-wip-us.apache.org/repos/asf/flink/blob/1a5930cb/test-infra/end-to-end-test/test_streaming_kafka010.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/test_streaming_kafka010.sh b/test-infra/end-to-end-test/test_streaming_kafka010.sh
new file mode 100755
index 0000000..638c34b
--- /dev/null
+++ b/test-infra/end-to-end-test/test_streaming_kafka010.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+set -e
+set -o pipefail
+
+# Convert relative path to absolute path
+TEST_ROOT=`pwd`
+TEST_INFRA_DIR="$0"
+TEST_INFRA_DIR=`dirname "$TEST_INFRA_DIR"`
+cd $TEST_INFRA_DIR
+TEST_INFRA_DIR=`pwd`
+cd $TEST_ROOT
+
+. "$TEST_INFRA_DIR"/common.sh
+
+
+start_cluster
+
+# get Kafka 0.10.0
+mkdir -p $TEST_DATA_DIR
+if [ -z "$3" ]; then
+ # need to download Kafka because no Kafka was specified on the invocation
+ KAFKA_URL="http://mirror.netcologne.de/apache.org/kafka/0.10.2.0/kafka_2.11-0.10.2.0.tgz"
+ echo "Downloading Kafka from $KAFKA_URL"
+ curl "$KAFKA_URL" > $TEST_DATA_DIR/kafka.tgz
+else
+ echo "Using specified Kafka from $3"
+ cp $3 $TEST_DATA_DIR/kafka.tgz
+fi
+
+tar xzf $TEST_DATA_DIR/kafka.tgz -C $TEST_DATA_DIR/
+KAFKA_DIR=$TEST_DATA_DIR/kafka_2.11-0.10.2.0
+
+# fix kafka config
+sed -i -e "s+^\(dataDir\s*=\s*\).*$+\1$TEST_DATA_DIR/zookeeper+" $KAFKA_DIR/config/zookeeper.properties
+sed -i -e "s+^\(log\.dirs\s*=\s*\).*$+\1$TEST_DATA_DIR/kafka+" $KAFKA_DIR/config/server.properties
+$KAFKA_DIR/bin/zookeeper-server-start.sh -daemon $KAFKA_DIR/config/zookeeper.properties
+$KAFKA_DIR/bin/kafka-server-start.sh -daemon $KAFKA_DIR/config/server.properties
+
+# zookeeper outputs the "Node does not exist" bit to stderr
+while [[ $($KAFKA_DIR/bin/zookeeper-shell.sh localhost:2181 get /brokers/ids/0 2>&1) =~ .*Node\ does\ not\ exist.* ]]; do
+ echo "Waiting for broker..."
+ sleep 1
+done
+
+# create the required topics
+$KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-input
+$KAFKA_DIR/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test-output
+
+# run the Flink job (detached mode)
+$FLINK_DIR/bin/flink run -d build-target/examples/streaming/Kafka010Example.jar \
+ --input-topic test-input --output-topic test-output \
+ --prefix=PREFIX \
+ --bootstrap.servers localhost:9092 --zookeeper.connect localhost:2181 --group.id myconsumer --auto.offset.reset earliest
+
+# send some data to Kafka
+echo -e "hello\nwhats\nup" | $KAFKA_DIR/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test-input
+
+# wait at most (roughly) 60 seconds until the results are there
+for i in {1..60}; do
+ DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --timeout-ms 0 2> /dev/null)
+
+ # make sure we have actual newlines in the string, not "\n"
+ EXPECTED=$(printf "PREFIX:hello\nPREFIX:whats\nPREFIX:up")
+
+ if [[ "$DATA_FROM_KAFKA" == "$EXPECTED" ]]; then
+ break
+ fi
+
+ echo "Waiting for results from Kafka..."
+ sleep 1
+done
+
+# verify again to set the PASS variable
+DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --timeout-ms 0 2> /dev/null)
+
+# make sure we have actual newlines in the string, not "\n"
+EXPECTED=$(printf "PREFIX:hello\nPREFIX:whats\nPREFIX:up")
+if [[ "$DATA_FROM_KAFKA" != "$EXPECTED" ]]; then
+ echo "Output from Flink program does not match expected output."
+ echo -e "EXPECTED: --$EXPECTED--"
+ echo -e "ACTUAL: --$DATA_FROM_KAFKA--"
+ PASS=""
+fi
+
+$KAFKA_DIR/bin/kafka-server-stop.sh
+$KAFKA_DIR/bin/zookeeper-server-stop.sh
+
+stop_cluster
+clean_data_dir
+check_all_pass
[4/4] flink git commit: [FLINK-6539] Run end-to-end tests on travis
Posted by al...@apache.org.
[FLINK-6539] Run end-to-end tests on travis
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/bb118104
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/bb118104
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/bb118104
Branch: refs/heads/master
Commit: bb118104bc71e00e097b799d1cadb7f95736f69b
Parents: 1a5930c
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Thu May 11 12:17:47 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Jul 24 11:20:04 2017 +0200
----------------------------------------------------------------------
tools/travis_mvn_watchdog.sh | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/bb118104/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 1c1172e..c2a51a5 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -274,7 +274,7 @@ check_shaded_artifacts() {
echo "=============================================================================="
return 1
fi
-
+
GUAVA=`cat allClasses | grep '^com/google/common' | wc -l`
if [ $GUAVA != "0" ]; then
echo "=============================================================================="
@@ -399,5 +399,17 @@ upload_artifacts_s3
# we are going back to
cd ../../
+# only run end-to-end tests in misc because we only have flink-dist here
+case $TEST in
+ (misc)
+ echo "Running automated end-to-end tests"
+
+ test-infra/end-to-end-test/test_batch_wordcount.sh build-target cluster
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ test-infra/end-to-end-test/test_streaming_kafka010.sh build-target cluster
+ EXIT_CODE=$(($EXIT_CODE+$?))
+ ;;
+esac
+
# Exit code for Travis build success/failure
exit $EXIT_CODE