You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2017/08/08 11:54:33 UTC

[1/3] flink git commit: [FLINK-7300] Fix unstable Kafka end-to-end tests

Repository: flink
Updated Branches:
  refs/heads/master 6f5fa7f74 -> 2ed74ca06


[FLINK-7300] Fix unstable Kafka end-to-end tests

Instead of specifying a timeout for the console consumer we now speciy
an uppper message boundary that matches our expected number of messages.
The timout was to unstable because it could sometimes return even though
messages were in fact available.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/65402e03
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/65402e03
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/65402e03

Branch: refs/heads/master
Commit: 65402e034c32e47824fe46427d83eb9c9ea22d30
Parents: 6f5fa7f
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Aug 7 18:08:03 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Aug 8 13:53:18 2017 +0200

----------------------------------------------------------------------
 .../end-to-end-test/test_streaming_kafka010.sh   | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/65402e03/test-infra/end-to-end-test/test_streaming_kafka010.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/test_streaming_kafka010.sh b/test-infra/end-to-end-test/test_streaming_kafka010.sh
index f358e24..cce8db4 100755
--- a/test-infra/end-to-end-test/test_streaming_kafka010.sh
+++ b/test-infra/end-to-end-test/test_streaming_kafka010.sh
@@ -73,24 +73,7 @@ $FLINK_DIR/bin/flink run -d build-target/examples/streaming/Kafka010Example.jar
 # send some data to Kafka
 echo -e "hello\nwhats\nup" | $KAFKA_DIR/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test-input
 
-# wait at most (roughly) 5 minutes until the results are there
-for i in {1..300}; do
-  DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --timeout-ms 0 2> /dev/null)
-
-  # make sure we have actual newlines in the string, not "\n"
-  EXPECTED=$(printf "PREFIX:hello\nPREFIX:whats\nPREFIX:up")
-
-  if [[ "$DATA_FROM_KAFKA" == "$EXPECTED" ]]; then
-    echo -e "Retrieved data from Kafka: --$DATA_FROM_KAFKA--"
-    break
-  fi
-
-  echo "Waiting for results from Kafka..."
-  sleep 1
-done
-
-# verify again to set the PASS variable
-DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --timeout-ms 0 2> /dev/null)
+DATA_FROM_KAFKA=$($KAFKA_DIR/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test-output --from-beginning --max-messages 3 2> /dev/null)
 
 # make sure we have actual newlines in the string, not "\n"
 EXPECTED=$(printf "PREFIX:hello\nPREFIX:whats\nPREFIX:up")


[3/3] flink git commit: [FLINK-7300] Add new Kafka errors to ignored errors in end-to-end tests

Posted by al...@apache.org.
[FLINK-7300] Add new Kafka errors to ignored errors in end-to-end tests

A new ERROR log message was added in FlinkKafkaConsumerBase that is
emitted when asynchronous offset commit to the Kafka broker fails. We
also ignore the accompanying exception, NoAvailableBrokersException.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2ed74ca0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2ed74ca0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2ed74ca0

Branch: refs/heads/master
Commit: 2ed74ca060ba64fefc7b53a23640d4854329f418
Parents: fd69dcc
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Aug 7 20:22:23 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Aug 8 13:54:22 2017 +0200

----------------------------------------------------------------------
 test-infra/end-to-end-test/common.sh | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/2ed74ca0/test-infra/end-to-end-test/common.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/common.sh b/test-infra/end-to-end-test/common.sh
index e05f70f..9cf5e7b 100644
--- a/test-infra/end-to-end-test/common.sh
+++ b/test-infra/end-to-end-test/common.sh
@@ -65,12 +65,20 @@ function stop_cluster {
     $FLINK_DIR/bin/stop-cluster.sh
   fi
 
-  if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log | grep -v "RetriableCommitFailedException" | grep -iq "error"; then
+  if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log \
+      | grep -v "RetriableCommitFailedException" \
+      | grep -v "NoAvailableBrokersException" \
+      | grep -v "Async Kafka commit failed" \
+      | grep -iq "error"; then
     echo "Found error in log files:"
     cat $FLINK_DIR/log/*
     PASS=""
   fi
-  if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log | grep -v "RetriableCommitFailedException" | grep -iq "exception"; then
+  if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log \
+      | grep -v "RetriableCommitFailedException" \
+      | grep -v "NoAvailableBrokersException" \
+      | grep -v "Async Kafka commit failed" \
+      | grep -iq "exception"; then
     echo "Found exception in log files:"
     cat $FLINK_DIR/log/*
     PASS=""


[2/3] flink git commit: [hotfix] Fix newline printing in travis_mvn_watchdog.sh

Posted by al...@apache.org.
[hotfix] Fix newline printing in travis_mvn_watchdog.sh


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/fd69dccc
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/fd69dccc
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/fd69dccc

Branch: refs/heads/master
Commit: fd69dcccba14d59b7b5226aac3a6a93be7d63804
Parents: 65402e0
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Mon Aug 7 18:12:29 2017 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Tue Aug 8 13:54:22 2017 +0200

----------------------------------------------------------------------
 tools/travis_mvn_watchdog.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/fd69dccc/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 557e1c9..fca3c7d 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -408,25 +408,25 @@ cd ../../
 case $TEST in
 	(misc)
 		if [ $EXIT_CODE == 0 ]; then
-			echo "\n=============================================================================="
-			echo "Running end-to-end tests"
-			echo "=============================================================================="
+			printf "\n\n==============================================================================\n"
+			printf "Running end-to-end tests\n"
+			printf "==============================================================================\n"
 
-			echo "\n=============================================================================="
-			echo "Running Wordcount end-to-end test"
-			echo "=============================================================================="
+			printf "\n==============================================================================\n"
+			printf "Running Wordcount end-to-end test\n"
+			printf "==============================================================================\n"
 			test-infra/end-to-end-test/test_batch_wordcount.sh build-target cluster
 			EXIT_CODE=$(($EXIT_CODE+$?))
 
-			echo "\n=============================================================================="
-			echo "Running Kafka end-to-end test"
-			echo "=============================================================================="
+			printf "\n==============================================================================\n"
+			printf "Running Kafka end-to-end test\n"
+			printf "==============================================================================\n"
 			test-infra/end-to-end-test/test_streaming_kafka010.sh build-target cluster
 			EXIT_CODE=$(($EXIT_CODE+$?))
 		else
-			echo "=============================================================================="
-			echo "Previous build failure detected, skipping end-to-end tests."
-			echo "=============================================================================="
+			printf "\n==============================================================================\n"
+			printf "Previous build failure detected, skipping end-to-end tests.\n"
+			printf "==============================================================================\n"
 		fi
 	;;
 esac