You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/03/09 12:27:46 UTC

flink git commit: [FLINK-8896] [kafka08] remove all cancel MARKERs before trying to find partition leaders

Repository: flink
Updated Branches:
  refs/heads/release-1.4 d57d4e0af -> 64db78c68


[FLINK-8896] [kafka08] remove all cancel MARKERs before trying to find partition leaders

This guards us against #cancel() being called multiple times and then trying to
look up an invalid topic/partition pair.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/64db78c6
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/64db78c6
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/64db78c6

Branch: refs/heads/release-1.4
Commit: 64db78c68368b22eff087eb4e74b62063420bc63
Parents: d57d4e0
Author: Nico Kruber <ni...@data-artisans.com>
Authored: Thu Mar 8 12:22:32 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Fri Mar 9 13:27:25 2018 +0100

----------------------------------------------------------------------
 .../streaming/connectors/kafka/internals/Kafka08Fetcher.java    | 3 ++-
 .../streaming/connectors/kafka/FlinkKafkaConsumerBase.java      | 5 ++++-
 2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/64db78c6/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
index d8bdd22..af95267 100644
--- a/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
+++ b/flink-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/Kafka08Fetcher.java
@@ -187,7 +187,8 @@ public class Kafka08Fetcher<T> extends AbstractFetcher<T, TopicAndPartition> {
 				// special marker into the queue
 				List<KafkaTopicPartitionState<TopicAndPartition>> partitionsToAssign =
 						unassignedPartitionsQueue.getBatchBlocking(5000);
-				partitionsToAssign.remove(MARKER);
+				// note: if there are more markers, remove them all
+				partitionsToAssign.removeIf(MARKER::equals);
 
 				if (!partitionsToAssign.isEmpty()) {
 					LOG.info("Assigning {} partitions to broker threads", partitionsToAssign.size());

http://git-wip-us.apache.org/repos/asf/flink/blob/64db78c6/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
----------------------------------------------------------------------
diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
index 537e73d..084f23d 100644
--- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaConsumerBase.java
@@ -626,7 +626,10 @@ public abstract class FlinkKafkaConsumerBase<T> extends RichParallelSourceFuncti
 						discoveryLoopErrorRef.set(e);
 					} finally {
 						// calling cancel will also let the fetcher loop escape
-						cancel();
+						// (if not running, cancel() was already called)
+						if (running) {
+							cancel();
+						}
 					}
 				}
 			});