You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2015/12/14 09:48:46 UTC

[2/2] camel git commit: CAMEL-9355: Reworked throttler implementation to be more performant and use a rolling window for time periods which gives a better flow. Thanks to Aaron Whiteside for the hard work and patch.

CAMEL-9355: Reworked throttler implementation to be more performant and use a rolling window for time periods which gives a better flow. Thanks to Aaron Whiteside for the hard work and patch.


Project: http://git-wip-us.apache.org/repos/asf/camel/repo
Commit: http://git-wip-us.apache.org/repos/asf/camel/commit/da5fa07b
Tree: http://git-wip-us.apache.org/repos/asf/camel/tree/da5fa07b
Diff: http://git-wip-us.apache.org/repos/asf/camel/diff/da5fa07b

Branch: refs/heads/master
Commit: da5fa07bd36ff74250cffda0bd58b7d03a24a997
Parents: beab07a
Author: Claus Ibsen <da...@apache.org>
Authored: Mon Dec 14 09:12:21 2015 +0100
Committer: Claus Ibsen <da...@apache.org>
Committed: Mon Dec 14 09:48:34 2015 +0100

----------------------------------------------------------------------
 .../management/mbean/ManagedThrottlerMBean.java |   3 -
 .../management/mbean/ManagedThrottler.java      |   4 -
 .../org/apache/camel/processor/Throttler.java   | 408 +++++++++++++------
 .../camel/management/ManagedThrottlerTest.java  |  38 --
 .../apache/camel/processor/ThrottlerTest.java   | 286 +++++--------
 5 files changed, 377 insertions(+), 362 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/camel/blob/da5fa07b/camel-core/src/main/java/org/apache/camel/api/management/mbean/ManagedThrottlerMBean.java
----------------------------------------------------------------------
diff --git a/camel-core/src/main/java/org/apache/camel/api/management/mbean/ManagedThrottlerMBean.java b/camel-core/src/main/java/org/apache/camel/api/management/mbean/ManagedThrottlerMBean.java
index 3bceae6..673c13e 100644
--- a/camel-core/src/main/java/org/apache/camel/api/management/mbean/ManagedThrottlerMBean.java
+++ b/camel-core/src/main/java/org/apache/camel/api/management/mbean/ManagedThrottlerMBean.java
@@ -32,9 +32,6 @@ public interface ManagedThrottlerMBean extends ManagedProcessorMBean {
     @ManagedAttribute(description = "Time period in millis")
     void setTimePeriodMillis(long timePeriodMillis);
 
-    @ManagedAttribute(description = "Number of exchanges currently throttled")
-    int getThrottledCount();
-
     @ManagedAttribute(description = "Enables asynchronous delay which means the thread will not block while delaying")
     Boolean isAsyncDelayed();
 

http://git-wip-us.apache.org/repos/asf/camel/blob/da5fa07b/camel-core/src/main/java/org/apache/camel/management/mbean/ManagedThrottler.java
----------------------------------------------------------------------
diff --git a/camel-core/src/main/java/org/apache/camel/management/mbean/ManagedThrottler.java b/camel-core/src/main/java/org/apache/camel/management/mbean/ManagedThrottler.java
index 0f49fb1..6c5e977 100644
--- a/camel-core/src/main/java/org/apache/camel/management/mbean/ManagedThrottler.java
+++ b/camel-core/src/main/java/org/apache/camel/management/mbean/ManagedThrottler.java
@@ -56,10 +56,6 @@ public class ManagedThrottler extends ManagedProcessor implements ManagedThrottl
         throttler.setTimePeriodMillis(timePeriodMillis);
     }
 
-    public int getThrottledCount() {
-        return throttler.getDelayedCount();
-    }
-
     public Boolean isAsyncDelayed() {
         return throttler.isAsyncDelayed();
     }

http://git-wip-us.apache.org/repos/asf/camel/blob/da5fa07b/camel-core/src/main/java/org/apache/camel/processor/Throttler.java
----------------------------------------------------------------------
diff --git a/camel-core/src/main/java/org/apache/camel/processor/Throttler.java b/camel-core/src/main/java/org/apache/camel/processor/Throttler.java
index 339da9d..dda02ec 100644
--- a/camel-core/src/main/java/org/apache/camel/processor/Throttler.java
+++ b/camel-core/src/main/java/org/apache/camel/processor/Throttler.java
@@ -16,8 +16,11 @@
  */
 package org.apache.camel.processor;
 
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.camel.AsyncCallback;
 import org.apache.camel.CamelContext;
@@ -27,7 +30,10 @@ import org.apache.camel.Processor;
 import org.apache.camel.RuntimeExchangeException;
 import org.apache.camel.Traceable;
 import org.apache.camel.spi.IdAware;
+import org.apache.camel.util.AsyncProcessorHelper;
 import org.apache.camel.util.ObjectHelper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A <a href="http://camel.apache.org/throttler.html">Throttler</a>
@@ -38,20 +44,46 @@ import org.apache.camel.util.ObjectHelper;
  * particular system to malfunction or to reduce its throughput you might want
  * to introduce some throttling.
  *
+ * This throttle implementation is thread-safe and is therefore safe to be used
+ * by multiple concurrent threads in a single route.
+ *
+ * The throttling mechanism is a DelayQueue with maxRequestsPerPeriod permits on
+ * it. Each permit is set to be delayed by timePeriodMillis (except when the
+ * throttler is initialized or the throttle rate increased, then there is no delay
+ * for those permits). Callers trying to acquire a permit from the DelayQueue will
+ * block if necessary. The end result is a rolling window of time. Where from the
+ * callers point of view in the last timePeriodMillis no more than
+ * maxRequestsPerPeriod have been allowed to be acquired.
+ *
  * @version
  */
-public class Throttler extends DelayProcessorSupport implements Traceable, IdAware {
+public class Throttler extends DelegateAsyncProcessor implements Traceable, IdAware {
+
+    private static final String PROPERTY_EXCHANGE_QUEUED_TIMESTAMP = "CamelThrottlerExchangeQueuedTimestamp";
+    private static final String PROPERTY_EXCHANGE_STATE = "CamelThrottlerExchangeState";
+
+    private enum State { SYNC, ASYNC, ASYNC_REJECTED }
+
+    private final Logger log = LoggerFactory.getLogger(Throttler.class);
+    private final CamelContext camelContext;
+    private final DelayQueue<ThrottlePermit> delayQueue = new DelayQueue<>();
+    private final ExecutorService asyncExecutor;
+    private final boolean shutdownAsyncExecutor;
+
+    private volatile long timePeriodMillis;
+    private volatile int throttleRate;
     private String id;
-    private volatile long maximumRequestsPerPeriod;
     private Expression maxRequestsPerPeriodExpression;
-    private AtomicLong timePeriodMillis = new AtomicLong(1000);
-    private volatile TimeSlot slot;
     private boolean rejectExecution;
+    private boolean asyncDelayed;
+    private boolean callerRunsWhenRejected = true;
 
-    public Throttler(CamelContext camelContext, Processor processor, Expression maxRequestsPerPeriodExpression, long timePeriodMillis,
-                     ScheduledExecutorService executorService, boolean shutdownExecutorService, boolean rejectExecution) {
-        super(camelContext, processor, executorService, shutdownExecutorService);
+    public Throttler(final CamelContext camelContext, final Processor processor, final Expression maxRequestsPerPeriodExpression, final long timePeriodMillis,
+                     final ExecutorService asyncExecutor, final boolean shutdownAsyncExecutor, final boolean rejectExecution) {
+        super(processor);
+        this.camelContext = camelContext;
         this.rejectExecution = rejectExecution;
+        this.shutdownAsyncExecutor = shutdownAsyncExecutor;
 
         ObjectHelper.notNull(maxRequestsPerPeriodExpression, "maxRequestsPerPeriodExpression");
         this.maxRequestsPerPeriodExpression = maxRequestsPerPeriodExpression;
@@ -59,177 +91,291 @@ public class Throttler extends DelayProcessorSupport implements Traceable, IdAwa
         if (timePeriodMillis <= 0) {
             throw new IllegalArgumentException("TimePeriodMillis should be a positive number, was: " + timePeriodMillis);
         }
-        this.timePeriodMillis.set(timePeriodMillis);
+        this.timePeriodMillis = timePeriodMillis;
+        this.asyncExecutor = asyncExecutor;
     }
 
     @Override
-    public String toString() {
-        return "Throttler[requests: " + maxRequestsPerPeriodExpression + " per: " + timePeriodMillis + " (ms) to: "
-               + getProcessor() + "]";
-    }
-
-    public String getTraceLabel() {
-        return "throttle[" + maxRequestsPerPeriodExpression + " per: " + timePeriodMillis + "]";
-    }
-
-    public String getId() {
-        return id;
-    }
+    public boolean process(final Exchange exchange, final AsyncCallback callback) {
+        long queuedStart = 0;
+        if (log.isTraceEnabled()) {
+            queuedStart = exchange.getProperty(PROPERTY_EXCHANGE_QUEUED_TIMESTAMP, 0L, Long.class);
+            exchange.removeProperty(PROPERTY_EXCHANGE_QUEUED_TIMESTAMP);
+        }
+        State state = exchange.getProperty(PROPERTY_EXCHANGE_STATE, State.SYNC, State.class);
+        exchange.removeProperty(PROPERTY_EXCHANGE_STATE);
+        boolean doneSync = state == State.SYNC || state == State.ASYNC_REJECTED;
 
-    public void setId(String id) {
-        this.id = id;
-    }
+        try {
+            if (!isRunAllowed()) {
+                throw new RejectedExecutionException("Run is not allowed");
+            }
 
-    // Properties
-    // -----------------------------------------------------------------------
+            calculateAndSetMaxRequestsPerPeriod(exchange);
+            ThrottlePermit permit = delayQueue.poll();
+
+            if (permit == null) {
+                if (isRejectExecution()) {
+                    throw new ThrottlerRejectedExecutionException("Exceeded the max throttle rate of "
+                            + throttleRate + " within " + timePeriodMillis + "ms");
+                } else {
+                    // delegate to async pool
+                    if (isAsyncDelayed() && !exchange.isTransacted() && state == State.SYNC) {
+                        log.debug("Throttle rate exceeded but AsyncDelayed enabled, so queueing for async processing, exchangeId: {}", exchange.getExchangeId());
+                        return processAsynchronously(exchange, callback);
+                    }
+
+                    // block waiting for a permit
+                    long start = 0;
+                    long elapsed = 0;
+                    if (log.isTraceEnabled()) {
+                        start = System.currentTimeMillis();
+                    }
+                    permit = delayQueue.take();
+                    if (log.isTraceEnabled()) {
+                        elapsed = System.currentTimeMillis() - start;
+                    }
+                    enqueuePermit(permit, exchange);
+
+                    if (state == State.ASYNC) {
+                        if (log.isTraceEnabled()) {
+                            long queuedTime = start - queuedStart;
+                            log.trace("Queued for {}ms, Throttled for {}ms, exchangeId: {}", queuedTime, elapsed, exchange.getExchangeId());
+                        }
+                    } else {
+                        log.trace("Throttled for {}ms, exchangeId: {}", elapsed, exchange.getExchangeId());
+                    }
+                }
+            } else {
+                enqueuePermit(permit, exchange);
+
+                if (state == State.ASYNC) {
+                    if (log.isTraceEnabled()) {
+                        long queuedTime = System.currentTimeMillis() - queuedStart;
+                        log.trace("Queued for {}ms, No throttling applied (throttle cleared while queued), for exchangeId: {}", queuedTime, exchange.getExchangeId());
+                    }
+                } else {
+                    log.trace("No throttling applied to exchangeId: {}", exchange.getExchangeId());
+                }
+            }
 
-    /**
-     * Sets the maximum number of requests per time period expression
-     */
-    public void setMaximumRequestsPerPeriodExpression(Expression maxRequestsPerPeriodExpression) {
-        this.maxRequestsPerPeriodExpression = maxRequestsPerPeriodExpression;
-    }
+            if (processor != null) {
+                if (doneSync) {
+                    return processor.process(exchange, callback);
+                } else {
+                    // if we are executing async, then we have to call the nested processor synchronously, and we
+                    // must not share our AsyncCallback, because the nested processing has no way of knowing that
+                    // we are already executing asynchronously.
+                    AsyncProcessorHelper.process(processor, exchange);
+                }
+            }
 
-    public Expression getMaximumRequestsPerPeriodExpression() {
-        return maxRequestsPerPeriodExpression;
-    }
+            callback.done(doneSync);
+            return doneSync;
 
-    public long getTimePeriodMillis() {
-        return timePeriodMillis.get();
+        } catch (final Throwable t) {
+            exchange.setException(t);
+            callback.done(doneSync);
+            return doneSync;
+        }
     }
 
     /**
-     * Gets the current maximum request per period value.
+     * Delegate blocking on the DelayQueue to an asyncExecutor. Except if the executor rejects the submission
+     * and isCallerRunsWhenRejected() is enabled, then this method will delegate back to process(), but not
+     * before changing the exchange state to stop any recursion.
      */
-    public long getCurrentMaximumRequestsPerPeriod() {
-        return maximumRequestsPerPeriod;
+    protected boolean processAsynchronously(final Exchange exchange, final AsyncCallback callback) {
+        try {
+            if (log.isTraceEnabled()) {
+                exchange.setProperty(PROPERTY_EXCHANGE_QUEUED_TIMESTAMP, System.currentTimeMillis());
+            }
+            exchange.setProperty(PROPERTY_EXCHANGE_STATE, State.ASYNC);
+            asyncExecutor.submit(new Runnable() {
+                @Override
+                public void run() {
+                    process(exchange, callback);
+                }
+            });
+            return false;
+        } catch (final RejectedExecutionException e) {
+            if (isCallerRunsWhenRejected()) {
+                log.debug("AsyncExecutor is full, rejected exchange will run in the current thread, exchangeId: {}", exchange.getExchangeId());
+                exchange.setProperty(PROPERTY_EXCHANGE_STATE, State.ASYNC_REJECTED);
+                return process(exchange, callback);
+            }
+            throw e;
+        }
     }
 
     /**
-     * Sets the time period during which the maximum number of requests apply
+     * Returns a permit to the DelayQueue, first resetting it's delay to be relative to now.
      */
-    public void setTimePeriodMillis(long timePeriodMillis) {
-        this.timePeriodMillis.set(timePeriodMillis);
+    protected void enqueuePermit(final ThrottlePermit permit, final Exchange exchange) {
+        permit.setDelayMs(getTimePeriodMillis());
+        delayQueue.put(permit);
+        // try and incur the least amount of overhead while releasing permits back to the queue
+        if (log.isTraceEnabled()) {
+            log.trace("Permit released, for exchangeId: {}", exchange.getExchangeId());
+        }
     }
 
-    // Implementation methods
-    // -----------------------------------------------------------------------
+    /**
+     * Evaluates the maxRequestsPerPeriodExpression and adjusts the throttle rate up or down.
+     */
+    protected void calculateAndSetMaxRequestsPerPeriod(final Exchange exchange) throws Exception {
+        Integer newThrottle = maxRequestsPerPeriodExpression.evaluate(exchange, Integer.class);
 
-    protected long calculateDelay(Exchange exchange) {
-        // evaluate as Object first to see if we get any result at all
-        Object result = maxRequestsPerPeriodExpression.evaluate(exchange, Object.class);
-        if (maximumRequestsPerPeriod == 0 && result == null) {
-            throw new RuntimeExchangeException("The max requests per period expression was evaluated as null: " + maxRequestsPerPeriodExpression, exchange);
+        if (newThrottle != null && newThrottle < 0) {
+            throw new IllegalStateException("The maximumRequestsPerPeriod must be a positive number, was: " + newThrottle);
         }
 
-        // then must convert value to long
-        Long longValue = exchange.getContext().getTypeConverter().convertTo(Long.class, result);
-        if (longValue != null) {
-            // log if we changed max period after initial setting
-            if (maximumRequestsPerPeriod > 0 && longValue.longValue() != maximumRequestsPerPeriod) {
-                log.debug("Throttler changed maximum requests per period from {} to {}", maximumRequestsPerPeriod, longValue);
-            }
-            if (maximumRequestsPerPeriod > longValue) {
-                slot.capacity = 0;
+        synchronized (this) {
+            if (newThrottle == null && throttleRate == 0) {
+                throw new RuntimeExchangeException("The maxRequestsPerPeriodExpression was evaluated as null: " + maxRequestsPerPeriodExpression, exchange);
             }
-            maximumRequestsPerPeriod = longValue;
-        }
 
-        if (maximumRequestsPerPeriod <= 0) {
-            throw new IllegalStateException("The maximumRequestsPerPeriod must be a positive number, was: " + maximumRequestsPerPeriod);
+            if (newThrottle != null) {
+                if (newThrottle != throttleRate) {
+                    // decrease
+                    if (throttleRate > newThrottle) {
+                        int delta = throttleRate - newThrottle;
+
+                        // discard any permits that are needed to decrease throttling
+                        while (delta > 0) {
+                            delayQueue.take();
+                            delta--;
+                            log.trace("Permit discarded due to throttling rate decrease, triggered by ExchangeId: {}", exchange.getExchangeId());
+                        }
+                        log.debug("Throttle rate decreased from {} to {}, triggered by ExchangeId: {}", throttleRate, newThrottle, exchange.getExchangeId());
+
+                    // increase
+                    } else if (newThrottle > throttleRate) {
+                        int delta = newThrottle - throttleRate;
+                        for (int i = 0; i < delta; i++) {
+                            delayQueue.put(new ThrottlePermit(-1));
+                        }
+                        if (throttleRate == 0) {
+                            log.debug("Initial throttle rate set to {}, triggered by ExchangeId: {}", newThrottle, exchange.getExchangeId());
+                        } else {
+                            log.debug("Throttle rate increase from {} to {}, triggered by ExchangeId: {}", throttleRate, newThrottle, exchange.getExchangeId());
+                        }
+                    }
+                    throttleRate = newThrottle;
+                }
+            }
         }
+    }
 
-        TimeSlot slot = nextSlot();
-        if (!slot.isActive()) {
-            long delay = slot.startTime - currentSystemTime();
-            return delay;
-        } else {
-            return 0;
+    @Override
+    protected void doStart() throws Exception {
+        if (isAsyncDelayed()) {
+            ObjectHelper.notNull(asyncExecutor, "executorService", this);
         }
+        super.doStart();
     }
 
-    /*
-     * Determine what the next available time slot is for handling an Exchange
-     */
-    protected synchronized TimeSlot nextSlot() throws ThrottlerRejectedExecutionException {
-        if (slot == null) {
-            slot = new TimeSlot();
-        } else {
-            if (rejectExecution && slot.isFull() && !slot.isPast()) {
-                throw new ThrottlerRejectedExecutionException("Exceed the max request limit!");
-            }
-            if (slot.isFull() || slot.isPast()) {
-                slot = slot.next();
-            }
+    @Override
+    protected void doShutdown() throws Exception {
+        if (shutdownAsyncExecutor && asyncExecutor != null) {
+            camelContext.getExecutorServiceManager().shutdownNow(asyncExecutor);
         }
-        slot.assign();
-        return slot;
+        super.doShutdown();
     }
 
-    /*
-    * A time slot is capable of handling a number of exchanges within a certain period of time.
-    */
-    protected class TimeSlot {
-
-        private volatile long capacity = Throttler.this.maximumRequestsPerPeriod;
-        private final long duration = Throttler.this.timePeriodMillis.get();
-        private final long startTime;
+    /**
+     * Permit that implements the Delayed interface needed by DelayQueue.
+     */
+    private class ThrottlePermit implements Delayed {
+        private volatile long scheduledTime;
 
-        protected TimeSlot() {
-            this(System.currentTimeMillis());
+        public ThrottlePermit(final long delayMs) {
+            setDelayMs(delayMs);
         }
 
-        protected TimeSlot(long startTime) {
-            this.startTime = startTime;
+        public void setDelayMs(final long delayMs) {
+            this.scheduledTime = System.currentTimeMillis() + delayMs;
         }
 
-        protected void assign() {
-            capacity--;
+        @Override
+        public long getDelay(final TimeUnit unit) {
+            return unit.convert(scheduledTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
         }
 
-        /*
-         * Start the next time slot either now or in the future
-         * (no time slots are being created in the past)
-         */
-        protected TimeSlot next() {
-            return new TimeSlot(Math.max(System.currentTimeMillis(), this.startTime + this.duration));
+        @Override
+        public int compareTo(final Delayed o) {
+            return (int)(getDelay(TimeUnit.MILLISECONDS) - o.getDelay(TimeUnit.MILLISECONDS));
         }
+    }
 
-        protected boolean isPast() {
-            long current = System.currentTimeMillis();
-            return current > (startTime + duration);
-        }
+    public boolean isRejectExecution() {
+        return rejectExecution;
+    }
 
-        protected boolean isActive() {
-            long current = System.currentTimeMillis();
-            return startTime <= current && current < (startTime + duration);
-        }
+    public void setRejectExecution(boolean rejectExecution) {
+        this.rejectExecution = rejectExecution;
+    }
 
-        protected boolean isFull() {
-            return capacity <= 0;
-        }
+    public boolean isAsyncDelayed() {
+        return asyncDelayed;
     }
 
-    TimeSlot getSlot() {
-        return this.slot;
+    public void setAsyncDelayed(boolean asyncDelayed) {
+        this.asyncDelayed = asyncDelayed;
     }
 
-    public boolean isRejectExecution() {
-        return rejectExecution;
+    public boolean isCallerRunsWhenRejected() {
+        return callerRunsWhenRejected;
     }
 
-    public void setRejectExecution(boolean rejectExecution) {
-        this.rejectExecution = rejectExecution;
+    public void setCallerRunsWhenRejected(boolean callerRunsWhenRejected) {
+        this.callerRunsWhenRejected = callerRunsWhenRejected;
     }
-    
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(final String id) {
+        this.id = id;
+    }
+
+    /**
+     * Sets the maximum number of requests per time period expression
+     */
+    public void setMaximumRequestsPerPeriodExpression(Expression maxRequestsPerPeriodExpression) {
+        this.maxRequestsPerPeriodExpression = maxRequestsPerPeriodExpression;
+    }
+
+    public Expression getMaximumRequestsPerPeriodExpression() {
+        return maxRequestsPerPeriodExpression;
+    }
+
+    /**
+     * Gets the current maximum request per period value.
+     */
+    public int getCurrentMaximumRequestsPerPeriod() {
+        return throttleRate;
+    }
+
+    /**
+     * Sets the time period during which the maximum number of requests apply
+     */
+    public void setTimePeriodMillis(final long timePeriodMillis) {
+        this.timePeriodMillis = timePeriodMillis;
+    }
+
+    public long getTimePeriodMillis() {
+        return timePeriodMillis;
+    }
+
+    public String getTraceLabel() {
+        return "throttle[" + maxRequestsPerPeriodExpression + " per: " + timePeriodMillis + "]";
+    }
+
     @Override
-    protected boolean processDelay(Exchange exchange, AsyncCallback callback, long delay) {
-        if (isRejectExecution() && delay > 0) {
-            exchange.setException(new ThrottlerRejectedExecutionException("Exceed the max request limit!"));
-            callback.done(true);
-            return true;
-        } else {
-            return super.processDelay(exchange, callback, delay);
-        }
+    public String toString() {
+        return "Throttler[requests: " + maxRequestsPerPeriodExpression + " per: " + timePeriodMillis + " (ms) to: "
+                + getProcessor() + "]";
     }
 }

http://git-wip-us.apache.org/repos/asf/camel/blob/da5fa07b/camel-core/src/test/java/org/apache/camel/management/ManagedThrottlerTest.java
----------------------------------------------------------------------
diff --git a/camel-core/src/test/java/org/apache/camel/management/ManagedThrottlerTest.java b/camel-core/src/test/java/org/apache/camel/management/ManagedThrottlerTest.java
index 5570e16..1f2cc18 100644
--- a/camel-core/src/test/java/org/apache/camel/management/ManagedThrottlerTest.java
+++ b/camel-core/src/test/java/org/apache/camel/management/ManagedThrottlerTest.java
@@ -135,19 +135,10 @@ public class ManagedThrottlerTest extends ManagementTestSupport {
         }
 
         assertTrue(notifier.matches(2, TimeUnit.SECONDS));
-        Integer throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-
-        // we are expecting this to be > 0
-        assertTrue(throttledMessages.intValue() > 0);
-
         assertMockEndpointsSatisfied();
 
-        throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-        assertEquals("Should not be any throttled messages left, found: " + throttledMessages, (Integer) 0, throttledMessages);
-
         Long completed = (Long) mbeanServer.getAttribute(routeName, "ExchangesCompleted");
         assertEquals(10, completed.longValue());
-
     }
 
     public void testThrottleAsyncVisableViaJmx() throws Exception {
@@ -183,19 +174,10 @@ public class ManagedThrottlerTest extends ManagementTestSupport {
         }
 
         assertTrue(notifier.matches(2, TimeUnit.SECONDS));
-        Integer throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-
-        // we are expecting this to be > 0
-        assertTrue(throttledMessages.intValue() > 0);
-
         assertMockEndpointsSatisfied();
 
-        throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-        assertEquals("Should not be any throttled messages left, found: " + throttledMessages, (Integer)0, throttledMessages);
-
         Long completed = (Long) mbeanServer.getAttribute(routeName, "ExchangesCompleted");
         assertEquals(10, completed.longValue());
-
     }
 
     public void testThrottleAsyncExceptionVisableViaJmx() throws Exception {
@@ -229,23 +211,14 @@ public class ManagedThrottlerTest extends ManagementTestSupport {
         }
 
         assertTrue(notifier.matches(2, TimeUnit.SECONDS));
-        Integer throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-
-        // we are expecting this to be > 0
-        assertTrue(throttledMessages.intValue() > 0);
-
         assertMockEndpointsSatisfied();
 
         // give a sec for exception handling to finish..
         Thread.sleep(500);
 
-        throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-        assertEquals("Should not be any throttled messages left, found: " + throttledMessages, (Integer)0, throttledMessages);
-
         // since all exchanges ended w/ exception, they are not completed
         Long completed = (Long) mbeanServer.getAttribute(routeName, "ExchangesCompleted");
         assertEquals(0, completed.longValue());
-
     }
 
     public void testRejectedExecution() throws Exception {
@@ -276,17 +249,11 @@ public class ManagedThrottlerTest extends ManagementTestSupport {
         MockEndpoint exceptionMock = getMockEndpoint("mock:rejectedExceptionEndpoint1");
         exceptionMock.expectedMessageCount(9);
 
-
         for (int i = 0; i < 10; i++) {
             template.sendBody("seda:throttleCountRejectExecution", "Message " + i);
         }
 
         assertMockEndpointsSatisfied();
-
-        // we shouldn't have ane leaked throttler counts
-        Integer throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-        assertEquals("Should not be any throttled messages left, found: " + throttledMessages, (Integer) 0, throttledMessages);
-
     }
 
     public void testRejectedExecutionCallerRuns() throws Exception {
@@ -317,16 +284,11 @@ public class ManagedThrottlerTest extends ManagementTestSupport {
         MockEndpoint exceptionMock = getMockEndpoint("mock:rejectedExceptionEndpoint");
         exceptionMock.expectedMessageCount(0);
 
-
         for (int i = 0; i < 10; i++) {
             template.sendBody("seda:throttleCountRejectExecutionCallerRuns", "Message " + i);
         }
 
         assertMockEndpointsSatisfied();
-
-        // we shouldn't have ane leaked throttler counts
-        Integer throttledMessages = (Integer) mbeanServer.getAttribute(throttlerName, "ThrottledCount");
-        assertEquals("Should not be any throttled messages left, found: " + throttledMessages, (Integer) 0, throttledMessages);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/camel/blob/da5fa07b/camel-core/src/test/java/org/apache/camel/processor/ThrottlerTest.java
----------------------------------------------------------------------
diff --git a/camel-core/src/test/java/org/apache/camel/processor/ThrottlerTest.java b/camel-core/src/test/java/org/apache/camel/processor/ThrottlerTest.java
index 3c4358e..527c052 100644
--- a/camel-core/src/test/java/org/apache/camel/processor/ThrottlerTest.java
+++ b/camel-core/src/test/java/org/apache/camel/processor/ThrottlerTest.java
@@ -18,26 +18,23 @@ package org.apache.camel.processor;
 
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
-import org.apache.camel.AsyncCallback;
 import org.apache.camel.ContextTestSupport;
 import org.apache.camel.builder.RouteBuilder;
 import org.apache.camel.component.mock.MockEndpoint;
-import org.apache.camel.impl.DefaultExchange;
-import org.apache.camel.processor.Throttler.TimeSlot;
-
-import static org.apache.camel.builder.Builder.constant;
 
 public class ThrottlerTest extends ContextTestSupport {
     private static final int INTERVAL = 500;
-    protected int messageCount = 9;
+    private static final int TOLERANCE = 50;
+    private static final int MESSAGE_COUNT = 9;
 
     protected boolean canTest() {
         // skip test on windows as it does not run well there
         return !isPlatform("windows");
     }
 
-    public void testSendLotsOfMessagesButOnly3GetThrough() throws Exception {
+    public void testSendLotsOfMessagesButOnly3GetThroughWithin2Seconds() throws Exception {
         if (!canTest()) {
             return;
         }
@@ -46,7 +43,7 @@ public class ThrottlerTest extends ContextTestSupport {
         resultEndpoint.expectedMessageCount(3);
         resultEndpoint.setResultWaitTime(2000);
 
-        for (int i = 0; i < messageCount; i++) {
+        for (int i = 0; i < MESSAGE_COUNT; i++) {
             template.sendBody("seda:a", "<message>" + i + "</message>");
         }
 
@@ -54,8 +51,18 @@ public class ThrottlerTest extends ContextTestSupport {
         // to check that the throttle really does kick in
         resultEndpoint.assertIsSatisfied();
     }
+
+//    public void testNoNestedProcessor() throws Exception {
+//        if (!canTest()) {
+//            return;
+//        }
+//
+//        MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
+//        long elapsed = sendMessagesAndAwaitDelivery(MESSAGE_COUNT, "direct:noNestedProcessor", MESSAGE_COUNT, resultEndpoint);
+//        assertThrottlerTiming(elapsed, 1, INTERVAL, MESSAGE_COUNT);
+//    }
     
-    public void testSendLotsOfMessagesWithRejctExecution() throws Exception {
+    public void testSendLotsOfMessagesWithRejectExecution() throws Exception {
         if (!canTest()) {
             return;
         }
@@ -84,236 +91,138 @@ public class ThrottlerTest extends ContextTestSupport {
         }
 
         MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
-        resultEndpoint.expectedMessageCount(messageCount);
-
-        ExecutorService executor = Executors.newFixedThreadPool(messageCount);
-
-        long start = System.currentTimeMillis();
-        for (int i = 0; i < messageCount; i++) {
-            executor.execute(new Runnable() {
-                public void run() {
-                    template.sendBody("direct:a", "<message>payload</message>");
-                }
-            });
-        }
-
-        // let's wait for the exchanges to arrive
-        resultEndpoint.assertIsSatisfied();
-
-        // now assert that they have actually been throttled
-        long minimumTime = (messageCount - 1) * INTERVAL;
-        // add a little slack
-        long delta = System.currentTimeMillis() - start + 750;
-        assertTrue("Should take at least " + minimumTime + "ms, was: " + delta, delta >= minimumTime);
-        executor.shutdownNow();
+        long elapsed = sendMessagesAndAwaitDelivery(MESSAGE_COUNT, "direct:a", MESSAGE_COUNT, resultEndpoint);
+        assertThrottlerTiming(elapsed, 1, INTERVAL, MESSAGE_COUNT);
     }
 
-    public void testTimeSlotCalculus() throws Exception {
+    public void testConfigurationWithConstantExpression() throws Exception {
         if (!canTest()) {
             return;
         }
 
-        Throttler throttler = new Throttler(context, null, constant(3), 1000, null, false, false);
-        // calculate will assign a new slot
-        throttler.calculateDelay(new DefaultExchange(context));
-        TimeSlot slot = throttler.nextSlot();
-        // start a new time slot
-        assertNotNull(slot);
-        // make sure the same slot is used (3 exchanges per slot)
-        assertSame(slot, throttler.nextSlot());
-        assertTrue(slot.isFull());
-        assertTrue(slot.isActive());
-
-        TimeSlot next = throttler.nextSlot();
-        // now we should have a new slot that starts somewhere in the future
-        assertNotSame(slot, next);
-        assertFalse(next.isActive());
+        MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
+        long elapsed = sendMessagesAndAwaitDelivery(MESSAGE_COUNT, "direct:expressionConstant", MESSAGE_COUNT, resultEndpoint);
+        assertThrottlerTiming(elapsed, 1, INTERVAL, MESSAGE_COUNT);
     }
 
-    public void testTimeSlotCalculusForPeriod() throws InterruptedException {
+    public void testConfigurationWithHeaderExpression() throws Exception {
         if (!canTest()) {
             return;
         }
 
-        Throttler throttler = new Throttler(context, null, constant(3), 1000, null, false, false);
-        throttler.calculateDelay(new DefaultExchange(context));
-
-        TimeSlot slot = throttler.getSlot();
-        assertNotNull(slot);
-        assertSame(slot, throttler.nextSlot());
+        MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
+        resultEndpoint.expectedMessageCount(MESSAGE_COUNT);
 
-        // we've only used up two of three slots, but now we introduce a time delay
-        // so to make the slot not valid anymore
-        Thread.sleep((long) (1.5 * 1000));
-        assertFalse(slot.isActive());
-        assertNotSame(slot, throttler.nextSlot());
+        ExecutorService executor = Executors.newFixedThreadPool(MESSAGE_COUNT);
+        try {
+            sendMessagesWithHeaderExpression(executor, resultEndpoint, 1, INTERVAL, MESSAGE_COUNT);
+        } finally {
+            executor.shutdownNow();
+        }
     }
 
-    public void testConfigurationWithConstantExpression() throws Exception {
+    public void testConfigurationWithChangingHeaderExpression() throws Exception {
         if (!canTest()) {
             return;
         }
 
-        MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
-        resultEndpoint.expectedMessageCount(messageCount);
-
-        ExecutorService executor = Executors.newFixedThreadPool(messageCount);
+        ExecutorService executor = Executors.newFixedThreadPool(5);
+        try {
+            MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
+            sendMessagesWithHeaderExpression(executor, resultEndpoint, 1, INTERVAL, MESSAGE_COUNT);
+            Thread.sleep(INTERVAL + TOLERANCE); // sleep here to ensure the first throttle rate does not influence the next one.
 
-        long start = System.currentTimeMillis();
-        for (int i = 0; i < messageCount; i++) {
-            executor.execute(new Runnable() {
-                public void run() {
-                    template.sendBody("direct:expressionConstant", "<message>payload</message>");
-                }
-            });
-        }
+            resultEndpoint.reset();
+            sendMessagesWithHeaderExpression(executor, resultEndpoint, 10, INTERVAL, MESSAGE_COUNT);
+            Thread.sleep(INTERVAL + TOLERANCE); // sleep here to ensure the first throttle rate does not influence the next one.
 
-        // let's wait for the exchanges to arrive
-        resultEndpoint.assertIsSatisfied();
+            resultEndpoint.reset();
+            sendMessagesWithHeaderExpression(executor, resultEndpoint, 1, INTERVAL, MESSAGE_COUNT);
+            Thread.sleep(INTERVAL + TOLERANCE); // sleep here to ensure the first throttle rate does not influence the next one.
 
-        // now assert that they have actually been throttled
-        long minimumTime = (messageCount - 1) * INTERVAL;
-        // add a little slack
-        long delta = System.currentTimeMillis() - start + 200;
-        assertTrue("Should take at least " + minimumTime + "ms, was: " + delta, delta >= minimumTime);
-        executor.shutdownNow();
+            resultEndpoint.reset();
+            sendMessagesWithHeaderExpression(executor, resultEndpoint, 10, INTERVAL, MESSAGE_COUNT);
+        } finally {
+            executor.shutdownNow();
+        }
     }
 
-    public void testConfigurationWithHeaderExpression() throws Exception {
+    public void testHighThrottleRate() throws Exception {
         if (!canTest()) {
             return;
         }
 
+        int messageCount = 20000;
         MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
-        resultEndpoint.expectedMessageCount(messageCount);
-
-        ExecutorService executor = Executors.newFixedThreadPool(messageCount);
-
-        long start = System.currentTimeMillis();
-        for (int i = 0; i < messageCount; i++) {
-            executor.execute(new Runnable() {
-                public void run() {
-                    template.sendBodyAndHeader("direct:expressionHeader", "<message>payload</message>", "throttleValue", 1);
-                }
-            });
-        }
-
-        // let's wait for the exchanges to arrive
-        resultEndpoint.assertIsSatisfied();
-
-        // now assert that they have actually been throttled
-        long minimumTime = (messageCount - 1) * INTERVAL;
-        // add a little slack
-        long delta = System.currentTimeMillis() - start + 200;
-        assertTrue("Should take at least " + minimumTime + "ms, was: " + delta, delta >= minimumTime);
-        executor.shutdownNow();
+        long elapsed = sendMessagesAndAwaitDelivery(messageCount, "direct:highThrottleRate", 5, resultEndpoint);
+        assertThrottlerTiming(elapsed, 1000, INTERVAL, messageCount);
     }
 
-    public void testConfigurationWithChangingHeaderExpression() throws Exception {
-        if (!canTest()) {
-            return;
-        }
+    private void assertThrottlerTiming(final long elapsedTimeMs, final int throttle, final int intervalMs, final int messageCount) {
+        // now assert that they have actually been throttled
+        long minimum = calculateMinimum(intervalMs, throttle, messageCount);
+        long maximum = calculateMaximum(intervalMs, throttle, messageCount);
+        log.info("Sent {} exchanges in {}ms, with throttle rate of {} per {}ms. Calculated min {}ms and max {}ms", new Object[]{messageCount, elapsedTimeMs, throttle, intervalMs, minimum, maximum});
 
-        ExecutorService executor = Executors.newFixedThreadPool(messageCount);
-        MockEndpoint resultEndpoint = resolveMandatoryEndpoint("mock:result", MockEndpoint.class);
-        sendMessagesWithHeaderExpression(executor, resultEndpoint, 1);
+        assertTrue("Should take at least " + minimum + "ms, was: " + elapsedTimeMs, elapsedTimeMs >= minimum);
+        assertTrue("Should take at most " + maximum + "ms, was: " + elapsedTimeMs, elapsedTimeMs <= maximum + TOLERANCE);
+    }
 
-        resultEndpoint.reset();
-        sendMessagesWithHeaderExpression(executor, resultEndpoint, 10);
+    private long sendMessagesAndAwaitDelivery(final int messageCount, final String endpointUri, final int threadPoolSize, final MockEndpoint receivingEndpoint) throws InterruptedException {
+        ExecutorService executor = Executors.newFixedThreadPool(threadPoolSize);
+        try {
+            if (receivingEndpoint != null) {
+                receivingEndpoint.expectedMessageCount(messageCount);
+            }
 
-        resultEndpoint.reset();
-        sendMessagesWithHeaderExpression(executor, resultEndpoint, 1);
+            long start = System.nanoTime();
+            for (int i = 0; i < messageCount; i++) {
+                executor.execute(new Runnable() {
+                    public void run() {
+                        template.sendBody(endpointUri, "<message>payload</message>");
+                    }
+                });
+            }
 
-        executor.shutdownNow();
+            // let's wait for the exchanges to arrive
+            if (receivingEndpoint != null) {
+                receivingEndpoint.assertIsSatisfied();
+            }
+            return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+        } finally {
+            executor.shutdownNow();
+        }
     }
 
-    private void sendMessagesWithHeaderExpression(ExecutorService executor, MockEndpoint resultEndpoint, final int
-            throttle) throws InterruptedException {
-
+    private void sendMessagesWithHeaderExpression(final ExecutorService executor, final MockEndpoint resultEndpoint, final int throttle, final int intervalMs, final int messageCount)
+            throws InterruptedException {
         resultEndpoint.expectedMessageCount(messageCount);
 
-        long start = System.currentTimeMillis();
+        long start = System.nanoTime();
         for (int i = 0; i < messageCount; i++) {
             executor.execute(new Runnable() {
                 public void run() {
-                    template.sendBodyAndHeader("direct:expressionHeader", "<message>payload</message>", "throttleValue",
-                            throttle);
+                    template.sendBodyAndHeader("direct:expressionHeader", "<message>payload</message>", "throttleValue", throttle);
                 }
             });
         }
 
         // let's wait for the exchanges to arrive
         resultEndpoint.assertIsSatisfied();
-
-        // now assert that they have actually been throttled
-        long minimumTime = (messageCount - 1) * INTERVAL / throttle;
-        // add a little slack
-        long delta = System.currentTimeMillis() - start + 200;
-        assertTrue("Should take at least " + minimumTime + "ms, was: " + delta, delta >= minimumTime);
-        long maxTime = (messageCount - 1) * INTERVAL / throttle * 3;
-        assertTrue("Should take at most " + maxTime + "ms, was: " + delta, delta <= maxTime);
+        long elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+        assertThrottlerTiming(elapsed, throttle, intervalMs, messageCount);
     }
 
-    /*
-    Given: you have a throttler which rejects messages
-    Then:
-    Throttler should return an exception when calculating the delay or set the exception
-    on the exchange when processing the delay.
-     */
-    public void testWhenTimeSlotIsFullShouldReturnThrottlerRejectedExecutionException() {
-        if (!canTest()) {
-            return;
-        }
-        Throttler throttler = new Throttler(context, null, constant(1), 1000, null, false, true);
-        AsyncCallback callback = new AsyncCallback() {
-            @Override
-            public void done(boolean doneSync) {
-
-            }
-        };
-        throttler.calculateDelay(new DefaultExchange(context));
-
-        boolean exceptionThrown = false;
-        DefaultExchange exchange = null;
-        try {
-            long delay = throttler.calculateDelay(new DefaultExchange(context));
-            exchange = new DefaultExchange(context);
-            throttler.processDelay(exchange,
-                    callback,
-                    delay);
-        } catch (ThrottlerRejectedExecutionException e) {
-            exceptionThrown = true;
+    private long calculateMinimum(final long periodMs, final long throttleRate, final long messageCount) {
+        if (messageCount % throttleRate > 0) {
+            return (long) Math.floor((double)messageCount / (double)throttleRate) * periodMs;
+        } else {
+            return (long) (Math.floor((double)messageCount / (double)throttleRate) * periodMs) - periodMs;
         }
-        assertTrue(exceptionThrown || exchange.getException().getClass() == ThrottlerRejectedExecutionException.class);
     }
 
-    /*
-       Given: you have a throttler which rejects messages after the first message
-       Then: the timeslot should be the original timeslot or the new timeslot should not be full
-     */
-    public void testRejectionOfExecutionShouldNotFillNextTimeSlot() {
-        if (!canTest()) {
-            return;
-        }
-        Throttler throttler = new Throttler(context, null, constant(1), 10000, null, false, true);
-        AsyncCallback callback = new AsyncCallback() {
-            @Override
-            public void done(boolean doneSync) {
-
-            }
-        };
-        throttler.calculateDelay(new DefaultExchange(context));
-        TimeSlot currentSlot = throttler.getSlot();
-        DefaultExchange exchange;
-        try {
-            long delay = throttler.calculateDelay(new DefaultExchange(context));
-            exchange = new DefaultExchange(context);
-            throttler.processDelay(exchange,
-                    callback,
-                    delay);
-        } catch (ThrottlerRejectedExecutionException ignore) {
-        }
-        assertTrue(currentSlot == throttler.getSlot() || !throttler.getSlot().isFull());
+    private long calculateMaximum(final long periodMs, final long throttleRate, final long messageCount) {
+        return ((long)Math.ceil((double)messageCount / (double)throttleRate)) * periodMs;
     }
 
     protected RouteBuilder createRouteBuilder() {
@@ -335,6 +244,11 @@ public class ThrottlerTest extends ContextTestSupport {
                 from("direct:expressionHeader").throttle(header("throttleValue")).timePeriodMillis(INTERVAL).to("log:result", "mock:result");
 
                 from("direct:start").throttle(2).timePeriodMillis(10000).rejectExecution(true).to("log:result", "mock:result");
+
+                from("direct:highThrottleRate").throttle(1000).timePeriodMillis(INTERVAL).to("mock:result");
+
+//                from("direct:noNestedProcessor").throttle(1).timePeriodMillis(INTERVAL).end().to("log:result", "mock:result");
+
             }
         };
     }