You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by iv...@apache.org on 2013/02/06 16:28:25 UTC
svn commit: r1443017 - in /zookeeper/bookkeeper/trunk: CHANGES.txt
hedwig-server/src/main/java/org/apache/hedwig/server/common/ServerConfiguration.java
hedwig-server/src/main/java/org/apache/hedwig/server/delivery/FIFODeliveryManager.java
Author: ivank
Date: Wed Feb 6 15:28:24 2013
New Revision: 1443017
URL: http://svn.apache.org/viewvc?rev=1443017&view=rev
Log:
BOOKKEEPER-526: multiple threads for delivery manager (sijie via ivank)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/common/ServerConfiguration.java
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/delivery/FIFODeliveryManager.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1443017&r1=1443016&r2=1443017&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Wed Feb 6 15:28:24 2013
@@ -12,6 +12,8 @@ Trunk (unreleased changes)
IMPROVEMENTS:
+ BOOKKEEPER-526: multiple threads for delivery manager (sijie via ivank)
+
BOOKKEEPER-544: Modify hedwig server tests to allow client testcases to start/stop them as part of their tests (mridul via ivank)
BOOKKEEPER-312: Implementation of JMS provider (mridul via ivank)
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/common/ServerConfiguration.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/common/ServerConfiguration.java?rev=1443017&r1=1443016&r2=1443017&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/common/ServerConfiguration.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/common/ServerConfiguration.java Wed Feb 6 15:28:24 2013
@@ -69,6 +69,7 @@ public class ServerConfiguration extends
protected final static String DEFAULT_MESSAGE_WINDOW_SIZE =
"default_message_window_size";
protected final static String NUM_READAHEAD_CACHE_THREADS = "num_readahead_cache_threads";
+ protected final static String NUM_DELIVERY_THREADS = "num_delivery_threads";
protected final static String MAX_ENTRIES_PER_LEDGER = "max_entries_per_ledger";
@@ -532,6 +533,15 @@ public class ServerConfiguration extends
}
/**
+ * Get number of delivery threads
+ *
+ * @return number of delivery threads.
+ */
+ public int getNumDeliveryThreads() {
+ return conf.getInt(NUM_DELIVERY_THREADS, Runtime.getRuntime().availableProcessors());
+ }
+
+ /**
* Whether enable metadata manager based topic manager.
*
* @return true if enabled metadata manager based topic manager.
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/delivery/FIFODeliveryManager.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/delivery/FIFODeliveryManager.java?rev=1443017&r1=1443016&r2=1443017&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/delivery/FIFODeliveryManager.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/delivery/FIFODeliveryManager.java Wed Feb 6 15:28:24 2013
@@ -18,7 +18,6 @@
package org.apache.hedwig.server.delivery;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Queue;
@@ -26,6 +25,8 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -63,7 +64,7 @@ import org.apache.hedwig.server.persiste
import org.apache.hedwig.util.Callback;
import static org.apache.hedwig.util.VarArgs.va;
-public class FIFODeliveryManager implements Runnable, DeliveryManager, SubChannelDisconnectedListener {
+public class FIFODeliveryManager implements DeliveryManager, SubChannelDisconnectedListener {
protected static final Logger logger = LoggerFactory.getLogger(FIFODeliveryManager.class);
@@ -81,49 +82,180 @@ public class FIFODeliveryManager impleme
}
/**
- * the main queue that the single-threaded delivery manager works off of
- */
- BlockingQueue<DeliveryManagerRequest> requestQueue = new LinkedBlockingQueue<DeliveryManagerRequest>();
-
- /**
- * The queue of all subscriptions that are facing a transient error either
- * in scanning from the persistence manager, or in sending to the consumer
- */
- Queue<ActiveSubscriberState> retryQueue =
- new PriorityBlockingQueue<ActiveSubscriberState>(32, new Comparator<ActiveSubscriberState>() {
- @Override
- public int compare(ActiveSubscriberState as1, ActiveSubscriberState as2) {
- long s = as1.lastScanErrorTime - as2.lastScanErrorTime;
- return s > 0 ? 1 : (s < 0 ? -1 : 0);
- }
- });
-
- /**
* Stores a mapping from topic to the delivery pointers on the topic. The
* delivery pointers are stored in a sorted map from seq-id to the set of
* subscribers at that seq-id
*/
- Map<ByteString, SortedMap<Long, Set<ActiveSubscriberState>>> perTopicDeliveryPtrs;
+ ConcurrentMap<ByteString, SortedMap<Long, Set<ActiveSubscriberState>>> perTopicDeliveryPtrs;
/**
* Mapping from delivery end point to the subscriber state that we are
* serving at that end point. This prevents us e.g., from serving two
* subscriptions to the same endpoint
*/
- Map<TopicSubscriber, ActiveSubscriberState> subscriberStates;
+ ConcurrentMap<TopicSubscriber, ActiveSubscriberState> subscriberStates;
private final ReadAheadCache cache;
private final PersistenceManager persistenceMgr;
private ServerConfiguration cfg;
- // Boolean indicating if this thread should continue running. This is used
- // when we want to stop the thread during a PubSubServer shutdown.
- protected boolean keepRunning = true;
- private final Thread workerThread;
+ private final int numDeliveryWorkers;
+ private final DeliveryWorker[] deliveryWorkers;
+
+ private class DeliveryWorker implements Runnable {
+
+ BlockingQueue<DeliveryManagerRequest> requestQueue =
+ new LinkedBlockingQueue<DeliveryManagerRequest>();;
+
+ /**
+ * The queue of all subscriptions that are facing a transient error either
+ * in scanning from the persistence manager, or in sending to the consumer
+ */
+ Queue<ActiveSubscriberState> retryQueue =
+ new PriorityBlockingQueue<ActiveSubscriberState>(32, new Comparator<ActiveSubscriberState>() {
+ @Override
+ public int compare(ActiveSubscriberState as1, ActiveSubscriberState as2) {
+ long s = as1.lastScanErrorTime - as2.lastScanErrorTime;
+ return s > 0 ? 1 : (s < 0 ? -1 : 0);
+ }
+ });
+
+ // Boolean indicating if this thread should continue running. This is used
+ // when we want to stop the thread during a PubSubServer shutdown.
+ protected volatile boolean keepRunning = true;
+ private final Thread workerThread;
+ private final int idx;
+
+ private Object suspensionLock = new Object();
+ private boolean suspended = false;
+
+ DeliveryWorker(int index) {
+ this.idx = index;
+ workerThread = new Thread(this, "DeliveryManagerThread-" + index);
+ }
+
+ void start() {
+ workerThread.start();
+ }
+
+ /**
+ * Stop method which will enqueue a ShutdownDeliveryManagerRequest.
+ */
+ void stop() {
+ enqueueWithoutFailure(new ShutdownDeliveryManagerRequest());
+ }
+
+ /**
+ * Stop FIFO delivery worker from processing requests. (for testing)
+ */
+ void suspendProcessing() {
+ synchronized(suspensionLock) {
+ suspended = true;
+ }
+ }
+
+ /**
+ * Resume FIFO delivery worker. (for testing)
+ */
+ void resumeProcessing() {
+ synchronized(suspensionLock) {
+ suspended = false;
+ suspensionLock.notify();
+ }
+ }
+
+ @Override
+ public void run() {
+ while (keepRunning) {
+ DeliveryManagerRequest request = null;
+
+ try {
+ // We use a timeout of 1 second, so that we can wake up once in
+ // a while to check if there is something in the retry queue.
+ request = requestQueue.poll(1, TimeUnit.SECONDS);
+ synchronized(suspensionLock) {
+ while (suspended) {
+ suspensionLock.wait();
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ // First retry any subscriptions that had failed and need a retry
+ retryErroredSubscribers();
+
+ if (request == null) {
+ continue;
+ }
+
+ request.performRequest();
+
+ }
+ }
+
+ protected void enqueueWithoutFailure(DeliveryManagerRequest request) {
+ if (!requestQueue.offer(request)) {
+ throw new UnexpectedError("Could not enqueue object: " + request
+ + " to request queue for delivery worker ." + idx);
+ }
+ }
+
+ public void retryErroredSubscriberAfterDelay(ActiveSubscriberState subscriber) {
+ subscriber.setLastScanErrorTime(MathUtils.now());
+
+ if (!retryQueue.offer(subscriber)) {
+ throw new UnexpectedError("Could not enqueue to retry queue for delivery worker " + idx);
+ }
+ getDeliveryWorker(subscriber.getTopic()).retryErroredSubscriberAfterDelay(subscriber);
+ }
+
+ public void clearRetryDelayForSubscriber(ActiveSubscriberState subscriber) {
+ subscriber.clearLastScanErrorTime();
+ if (!retryQueue.offer(subscriber)) {
+ throw new UnexpectedError("Could not enqueue to delivery manager retry queue");
+ }
+ // no request in request queue now
+ // issue a empty delivery request to not waiting for polling requests queue
+ if (requestQueue.isEmpty()) {
+ enqueueWithoutFailure(new DeliveryManagerRequest() {
+ @Override
+ public void performRequest() {
+ // do nothing
+ }
+ });
+ }
+ }
+
+ protected void retryErroredSubscribers() {
+ long lastInterestingFailureTime = MathUtils.now() - cfg.getScanBackoffPeriodMs();
+ ActiveSubscriberState subscriber;
+
+ while ((subscriber = retryQueue.peek()) != null) {
+ if (subscriber.getLastScanErrorTime() > lastInterestingFailureTime) {
+ // Not enough time has elapsed yet, will retry later
+ // Since the queue is fifo, no need to check later items
+ return;
+ }
+
+ // retry now
+ subscriber.deliverNextMessage();
+ retryQueue.poll();
+ }
+ }
+
+ protected class ShutdownDeliveryManagerRequest implements DeliveryManagerRequest {
+ // This is a simple type of Request we will enqueue when the
+ // PubSubServer is shut down and we want to stop the DeliveryManager
+ // thread.
+ public void performRequest() {
+ keepRunning = false;
+ }
+ }
+
+ }
- private Object suspensionLock = new Object();
- private boolean suspended = false;
public FIFODeliveryManager(PersistenceManager persistenceMgr, ServerConfiguration cfg) {
@@ -133,14 +265,23 @@ public class FIFODeliveryManager impleme
} else {
this.cache = null;
}
- perTopicDeliveryPtrs = new HashMap<ByteString, SortedMap<Long, Set<ActiveSubscriberState>>>();
- subscriberStates = new HashMap<TopicSubscriber, ActiveSubscriberState>();
- workerThread = new Thread(this, "DeliveryManagerThread");
+ perTopicDeliveryPtrs =
+ new ConcurrentHashMap<ByteString, SortedMap<Long, Set<ActiveSubscriberState>>>();
+ subscriberStates =
+ new ConcurrentHashMap<TopicSubscriber, ActiveSubscriberState>();
this.cfg = cfg;
+ // initialize the delivery workers
+ this.numDeliveryWorkers = cfg.getNumDeliveryThreads();
+ this.deliveryWorkers = new DeliveryWorker[numDeliveryWorkers];
+ for (int i=0; i<numDeliveryWorkers; i++) {
+ deliveryWorkers[i] = new DeliveryWorker(i);
+ }
}
public void start() {
- workerThread.start();
+ for (int i=0; i<numDeliveryWorkers; i++) {
+ deliveryWorkers[i].start();
+ }
}
/**
@@ -148,8 +289,8 @@ public class FIFODeliveryManager impleme
*/
@VisibleForTesting
public void suspendProcessing() {
- synchronized(suspensionLock) {
- suspended = true;
+ for (int i=0; i<numDeliveryWorkers; i++) {
+ deliveryWorkers[i].suspendProcessing();
}
}
@@ -158,28 +299,33 @@ public class FIFODeliveryManager impleme
*/
@VisibleForTesting
public void resumeProcessing() {
- synchronized(suspensionLock) {
- suspended = false;
- suspensionLock.notify();
+ for (int i=0; i<numDeliveryWorkers; i++) {
+ deliveryWorkers[i].resumeProcessing();
}
}
/**
- * ===================================================================== Our
- * usual enqueue function, stop if error because of unbounded queue, should
- * never happen
- *
+ * Stop the FIFO delivery manager.
*/
- protected void enqueueWithoutFailure(DeliveryManagerRequest request) {
- if (!requestQueue.offer(request)) {
- throw new UnexpectedError("Could not enqueue object: " + request + " to delivery manager request queue.");
+ public void stop() {
+ for (int i=0; i<numDeliveryWorkers; i++) {
+ deliveryWorkers[i].stop();
}
}
+ private DeliveryWorker getDeliveryWorker(ByteString topic) {
+ return deliveryWorkers[MathUtils.signSafeMod(topic.hashCode(), numDeliveryWorkers)];
+ }
+
/**
- * ====================================================================
- * Public interface of the delivery manager
+ * ===================================================================== Our
+ * usual enqueue function, stop if error because of unbounded queue, should
+ * never happen
+ *
*/
+ protected void enqueueWithoutFailure(ByteString topic, DeliveryManagerRequest request) {
+ getDeliveryWorker(topic).enqueueWithoutFailure(request);
+ }
/**
* Tells the delivery manager to start sending out messages for a particular
@@ -211,13 +357,13 @@ public class FIFODeliveryManager impleme
seqIdToStartFrom.getLocalComponent() - 1,
endPoint, filter, callback, ctx);
- enqueueWithoutFailure(subscriber);
+ enqueueWithoutFailure(topic, subscriber);
}
public void stopServingSubscriber(ByteString topic, ByteString subscriberId,
SubscriptionEvent event,
Callback<Void> cb, Object ctx) {
- enqueueWithoutFailure(new StopServingSubscriber(topic, subscriberId, event, cb, ctx));
+ enqueueWithoutFailure(topic, new StopServingSubscriber(topic, subscriberId, event, cb, ctx));
}
/**
@@ -227,29 +373,11 @@ public class FIFODeliveryManager impleme
* @param subscriber
*/
public void retryErroredSubscriberAfterDelay(ActiveSubscriberState subscriber) {
-
- subscriber.setLastScanErrorTime(MathUtils.now());
-
- if (!retryQueue.offer(subscriber)) {
- throw new UnexpectedError("Could not enqueue to delivery manager retry queue");
- }
+ getDeliveryWorker(subscriber.getTopic()).retryErroredSubscriberAfterDelay(subscriber);
}
public void clearRetryDelayForSubscriber(ActiveSubscriberState subscriber) {
- subscriber.clearLastScanErrorTime();
- if (!retryQueue.offer(subscriber)) {
- throw new UnexpectedError("Could not enqueue to delivery manager retry queue");
- }
- // no request in request queue now
- // issue a empty delivery request to not waiting for polling requests queue
- if (requestQueue.isEmpty()) {
- enqueueWithoutFailure(new DeliveryManagerRequest() {
- @Override
- public void performRequest() {
- // do nothing
- }
- });
- }
+ getDeliveryWorker(subscriber.getTopic()).clearRetryDelayForSubscriber(subscriber);
}
// TODO: for now, I don't move messageConsumed request to delivery manager thread,
@@ -274,64 +402,8 @@ public class FIFODeliveryManager impleme
* @param newSeqId
*/
public void moveDeliveryPtrForward(ActiveSubscriberState subscriber, long prevSeqId, long newSeqId) {
- enqueueWithoutFailure(new DeliveryPtrMove(subscriber, prevSeqId, newSeqId));
- }
-
- /*
- * ==========================================================================
- * == End of public interface, internal machinery begins.
- */
- public void run() {
- while (keepRunning) {
- DeliveryManagerRequest request = null;
-
- try {
- // We use a timeout of 1 second, so that we can wake up once in
- // a while to check if there is something in the retry queue.
- request = requestQueue.poll(1, TimeUnit.SECONDS);
- synchronized(suspensionLock) {
- while (suspended) {
- suspensionLock.wait();
- }
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
-
- // First retry any subscriptions that had failed and need a retry
- retryErroredSubscribers();
-
- if (request == null) {
- continue;
- }
-
- request.performRequest();
-
- }
- }
-
- /**
- * Stop method which will enqueue a ShutdownDeliveryManagerRequest.
- */
- public void stop() {
- enqueueWithoutFailure(new ShutdownDeliveryManagerRequest());
- }
-
- protected void retryErroredSubscribers() {
- long lastInterestingFailureTime = MathUtils.now() - cfg.getScanBackoffPeriodMs();
- ActiveSubscriberState subscriber;
-
- while ((subscriber = retryQueue.peek()) != null) {
- if (subscriber.getLastScanErrorTime() > lastInterestingFailureTime) {
- // Not enough time has elapsed yet, will retry later
- // Since the queue is fifo, no need to check later items
- return;
- }
-
- // retry now
- subscriber.deliverNextMessage();
- retryQueue.poll();
- }
+ enqueueWithoutFailure(subscriber.getTopic(),
+ new DeliveryPtrMove(subscriber, prevSeqId, newSeqId));
}
protected void removeDeliveryPtr(ActiveSubscriberState subscriber, Long seqId, boolean isAbsenceOk,
@@ -525,7 +597,7 @@ public class FIFODeliveryManager impleme
logger.info("Try to wake up subscriber ({}) to deliver messages again : last delivered {}, last consumed {}.",
va(this, lastLocalSeqIdDelivered, lastSeqIdConsumedUtil));
- enqueueWithoutFailure(new DeliveryManagerRequest() {
+ enqueueWithoutFailure(topic, new DeliveryManagerRequest() {
@Override
public void performRequest() {
// enqueue
@@ -843,15 +915,6 @@ public class FIFODeliveryManager impleme
}
}
- protected class ShutdownDeliveryManagerRequest implements DeliveryManagerRequest {
- // This is a simple type of Request we will enqueue when the
- // PubSubServer is shut down and we want to stop the DeliveryManager
- // thread.
- public void performRequest() {
- keepRunning = false;
- }
- }
-
/**
* ====================================================================
*