You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2016/02/19 19:40:58 UTC

[3/4] flink git commit: [FLINK-3368][Kafka 0.8] Some improvements to the Legacy Fetcher

[FLINK-3368][Kafka 0.8] Some improvements to the Legacy Fetcher


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/0b7c2db7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/0b7c2db7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/0b7c2db7

Branch: refs/heads/master
Commit: 0b7c2db76fd25b5cde7c14c7eb906c308a4ecfb9
Parents: 3a8717f
Author: Stephan Ewen <se...@apache.org>
Authored: Thu Feb 18 21:12:03 2016 +0100
Committer: Robert Metzger <rm...@apache.org>
Committed: Fri Feb 19 11:09:21 2016 +0100

----------------------------------------------------------------------
 .../kafka/internals/LegacyFetcher.java          | 132 ++++++++-----------
 1 file changed, 55 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/0b7c2db7/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
----------------------------------------------------------------------
diff --git a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
index e8ae931..a18b159 100644
--- a/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
+++ b/flink-streaming-connectors/flink-connector-kafka-0.8/src/main/java/org/apache/flink/streaming/connectors/kafka/internals/LegacyFetcher.java
@@ -64,6 +64,8 @@ public class LegacyFetcher implements Fetcher {
 	
 	private static final Logger LOG = LoggerFactory.getLogger(LegacyFetcher.class);
 
+	private static final FetchPartition MARKER = new FetchPartition("n/a", -1, -1);
+	
 	
 	/** The properties that configure the Kafka connection */
 	private final Properties config;
@@ -81,7 +83,7 @@ public class LegacyFetcher implements Fetcher {
 	/** Reference the the thread that executed the run() method. */
 	private volatile Thread mainThread;
 	
-	/** Flag to shot the fetcher down */
+	/** Flag to shut the fetcher down */
 	private volatile boolean running = true;
 
 	/**
@@ -105,22 +107,18 @@ public class LegacyFetcher implements Fetcher {
 		
 		this.config = requireNonNull(props, "The config properties cannot be null");
 		this.userCodeClassloader = requireNonNull(userCodeClassloader);
-		if(initialPartitionsToRead.size() == 0) {
+		if (initialPartitionsToRead.size() == 0) {
 			throw new IllegalArgumentException("List of initial partitions is empty");
 		}
 
-		try {
-			for(Map.Entry<KafkaTopicPartition, Long> partitionToRead: initialPartitionsToRead.entrySet()) {
-				KafkaTopicPartition ktp = partitionToRead.getKey();
-				// we increment the offset by one so that we fetch the next message in the partition.
-				long offset = partitionToRead.getValue();
-				if(offset >= 0 && offset != FlinkKafkaConsumer08.OFFSET_NOT_SET) {
-					offset += 1L;
-				}
-				unassignedPartitions.addIfOpen(new FetchPartition(ktp.getTopic(), ktp.getPartition(), offset));
+		for (Map.Entry<KafkaTopicPartition, Long> partitionToRead: initialPartitionsToRead.entrySet()) {
+			KafkaTopicPartition ktp = partitionToRead.getKey();
+			// we increment the offset by one so that we fetch the next message in the partition.
+			long offset = partitionToRead.getValue();
+			if (offset >= 0 && offset != FlinkKafkaConsumer08.OFFSET_NOT_SET) {
+				offset += 1L;
 			}
-		} catch (IllegalStateException e) {
-			throw new RuntimeException("Fetcher initialization got interrupted", e);
+			unassignedPartitions.add(new FetchPartition(ktp.getTopic(), ktp.getPartition(), offset));
 		}
 		this.taskName = taskName;
 		this.error = new AtomicReference<>();
@@ -149,81 +147,64 @@ public class LegacyFetcher implements Fetcher {
 		this.mainThread = Thread.currentThread();
 
 		// keep presumably dead threads in the list until we are sure the thread is not alive anymore.
-		List<SimpleConsumerThread<T>> deadBrokerThreads = new ArrayList<>();
-		Map<Node, SimpleConsumerThread<T>> brokerToThread = new HashMap<>();
+		final Map<Node, SimpleConsumerThread<T>> brokerToThread = new HashMap<>();
 		try {
 			// Main loop polling elements from the unassignedPartitions queue to the threads
 			while (running && error.get() == null) {
 				try {
 					// wait for 5 seconds trying to get partitions to assign
 					List<FetchPartition> partitionsToAssign = unassignedPartitions.getBatchBlocking(5000);
+					partitionsToAssign.remove(MARKER);
+					
 					if(!partitionsToAssign.isEmpty()) {
 						LOG.info("Assigning {} partitions to broker threads", partitionsToAssign.size());
 						Map<Node, List<FetchPartition>> partitionsWithLeaders = findLeaderForPartitions(partitionsToAssign);
-
+						
 						// assign the partitions to the leaders (maybe start the threads)
 						for (Map.Entry<Node, List<FetchPartition>> partitionsWithLeader : partitionsWithLeaders.entrySet()) {
 							final Node leader = partitionsWithLeader.getKey();
 							final List<FetchPartition> partitions = partitionsWithLeader.getValue();
 							SimpleConsumerThread<T> brokerThread = brokerToThread.get(leader);
-							// TODO: maybe add a check again here if we are still running.
-							if (brokerThread == null) {
+
+							if (!running) {
+								break;
+							}
+							
+							if (brokerThread == null || !brokerThread.getNewPartitionsQueue().isOpen()) {
 								// start new thread
 								brokerThread = createAndStartSimpleConsumerThread(sourceContext, deserializer, lastOffsets, partitions, leader);
 								brokerToThread.put(leader, brokerThread);
 							} else {
-								if(brokerThread.isAlive()) {
-									// put elements into queue of thread
-									ClosableBlockingQueue<FetchPartition> newPartitionsQueue = brokerThread.getNewPartitionsQueue();
-									for (FetchPartition fp : partitions) {
-										if (!newPartitionsQueue.addIfOpen(fp)) {
-											// we were unable to add the partition to the broker's queue
-											// the broker has closed in the meantime (the thread will shut down)
-											// create a new thread for connecting to this broker
-											List<FetchPartition> seedPartitions = new ArrayList<>();
-											seedPartitions.add(fp);
-											brokerThread = createAndStartSimpleConsumerThread(sourceContext, deserializer, lastOffsets, seedPartitions, leader);
-											SimpleConsumerThread<T> oldThread = brokerToThread.put(leader, brokerThread);
-											if(oldThread != null) {
-												deadBrokerThreads.add(oldThread);
-											}
-											newPartitionsQueue = brokerThread.getNewPartitionsQueue(); // update queue for the subsequent partitions
-										}
-									}
-								} else {
-									// broker shut down in the meantime. Start it:
-									brokerThread = createAndStartSimpleConsumerThread(sourceContext, deserializer, lastOffsets, partitions, leader);
-									SimpleConsumerThread<T> oldThread = brokerToThread.put(leader, brokerThread);
-									if(oldThread != null) {
-										deadBrokerThreads.add(oldThread);
+								// put elements into queue of thread
+								ClosableBlockingQueue<FetchPartition> newPartitionsQueue = brokerThread.getNewPartitionsQueue();
+								for (FetchPartition fp : partitions) {
+									if (!newPartitionsQueue.addIfOpen(fp)) {
+										// we were unable to add the partition to the broker's queue
+										// the broker has closed in the meantime (the thread will shut down)
+										// create a new thread for connecting to this broker
+										List<FetchPartition> seedPartitions = new ArrayList<>();
+										seedPartitions.add(fp);
+										brokerThread = createAndStartSimpleConsumerThread(sourceContext, deserializer, lastOffsets, seedPartitions, leader);
+										brokerToThread.put(leader, brokerThread);
+										newPartitionsQueue = brokerThread.getNewPartitionsQueue(); // update queue for the subsequent partitions
 									}
 								}
 							}
 						}
 					} else {
 						// there were no partitions to assign. Check if any broker threads shut down.
-						Iterator<Map.Entry<Node, SimpleConsumerThread<T>>> bttIterator = brokerToThread.entrySet().iterator();
-						while(bttIterator.hasNext()) {
-							Map.Entry<Node, SimpleConsumerThread<T>> brokerAndThread = bttIterator.next();
-							if(!brokerAndThread.getValue().isAlive()) {
-								LOG.info("Removing stopped consumer thread {}", brokerAndThread.getValue().getName());
+						Iterator<SimpleConsumerThread<T>> bttIterator = brokerToThread.values().iterator();
+						while (bttIterator.hasNext()) {
+							SimpleConsumerThread<T> thread = bttIterator.next();
+							if(!thread.getNewPartitionsQueue().isOpen()) {
+								LOG.info("Removing stopped consumer thread {}", thread.getName());
 								bttIterator.remove();
 							}
 						}
 					}
 
-					if(deadBrokerThreads.size() > 0) {
-						// see how the dead brokers are doing
-						Iterator<SimpleConsumerThread<T>> deadBrokerThreadsIterator = deadBrokerThreads.iterator();
-						while(deadBrokerThreadsIterator.hasNext()) {
-							if(!deadBrokerThreadsIterator.next().isAlive()) {
-								deadBrokerThreadsIterator.remove();
-							}
-						}
-					}
-
-					if(brokerToThread.size() == 0 && unassignedPartitions.peek() == null) {
-						if(unassignedPartitions.close()) {
+					if (brokerToThread.size() == 0 && unassignedPartitions.isEmpty()) {
+						if (unassignedPartitions.close()) {
 							LOG.info("All consumer threads are finished, there are no more unassigned partitions. Stopping fetcher");
 							break;
 						}
@@ -250,17 +231,6 @@ public class LegacyFetcher implements Fetcher {
 						runningThreads++;
 					}
 				}
-
-				// also ensure shutdown of dead brokers:
-				if (deadBrokerThreads.size() > 0) {
-					// see how the dead brokers are doing
-					for (SimpleConsumerThread<T> thread : deadBrokerThreads) {
-						if (thread.isAlive()) {
-							thread.cancel();
-							runningThreads++;
-						}
-					}
-				}
 				if(runningThreads > 0) {
 					Thread.sleep(500);
 				}
@@ -316,6 +286,7 @@ public class LegacyFetcher implements Fetcher {
 		// create new list to remove elements from
 		List<FetchPartition> partitionsToAssignInternal = new ArrayList<>(partitionsToAssign);
 		Map<Node, List<FetchPartition>> leaderToPartitions = new HashMap<>();
+		
 		for(KafkaTopicPartitionLeader partitionLeader: topicPartitionWithLeaderList) {
 			if(partitionsToAssignInternal.size() == 0) {
 				// we are done: all partitions are assigned
@@ -512,6 +483,11 @@ public class LegacyFetcher implements Fetcher {
 							// close succeeded. Closing thread
 							LOG.info("Consumer thread {} does not have any partitions assigned anymore. Stopping thread.", getName());
 							running = false;
+							
+							// add the wake-up marker into the queue to make the main thread
+							// immediately wake up and termination faster
+							unassignedPartitions.add(MARKER);
+							
 							break;
 						} else {
 							// close failed: LegacyFetcher main thread added new partitions into the queue.
@@ -544,10 +520,8 @@ public class LegacyFetcher implements Fetcher {
 							// retry a few times, then return ALL partitions for new leader lookup
 							if(++reconnects >= reconnectLimit) {
 								LOG.warn("Unable to reach broker after {} retries. Returning all current partitions", reconnectLimit);
-								for(FetchPartition fp: this.partitions) {
-									if(!unassignedPartitions.addIfOpen(fp)) {
-										throw new RuntimeException("Main thread closed unassigned partitions queue");
-									}
+								for (FetchPartition fp: this.partitions) {
+									unassignedPartitions.add(fp);
 								}
 								this.partitions.clear();
 								continue; // jump to top of loop: will close thread or subscribe to new partitions
@@ -593,9 +567,9 @@ public class LegacyFetcher implements Fetcher {
 								// the broker we are connected to is not the leader for the partition.
 								LOG.warn("{} is not the leader of {}. Reassigning leader for partition", broker, fp);
 								LOG.debug("Error code = {}", code);
-								if(!unassignedPartitions.addIfOpen(fp)) {
-									throw new RuntimeException("Main thread closed unassigned partitions queue");
-								}
+								
+								unassignedPartitions.add(fp);
+								
 								partitionsIterator.remove(); // unsubscribe the partition ourselves
 								partitionsRemoved = true;
 							} else if (code != ErrorMapping.NoError()) {
@@ -693,6 +667,10 @@ public class LegacyFetcher implements Fetcher {
 					}
 					LOG.debug("This fetch contained {} messages ({} deleted messages)", messagesInFetch, deletedMessages);
 				} // end of fetch loop
+				
+				if (!newPartitionsQueue.close()) {
+					throw new Exception("Bug: Cleanly leaving fetcher thread without having a closed queue.");
+				}
 			}
 			catch (Throwable t) {
 				// report to the main thread