You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2010/02/17 14:44:42 UTC
svn commit: r910970 - in /camel/trunk/camel-core/src:
main/java/org/apache/camel/model/
main/java/org/apache/camel/processor/aggregate/
test/java/org/apache/camel/processor/aggregator/
Author: davsclaus
Date: Wed Feb 17 13:44:42 2010
New Revision: 910970
URL: http://svn.apache.org/viewvc?rev=910970&view=rev
Log:
CAMEL-1686: Overhaul of aggregator.
Added:
camel/trunk/camel-core/src/test/java/org/apache/camel/processor/aggregator/AggregateConcurrentPerCorrelationKeyTest.java (with props)
Modified:
camel/trunk/camel-core/src/main/java/org/apache/camel/model/AggregateDefinition.java
camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java
camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/ClosedCorrelationKeyException.java
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/model/AggregateDefinition.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/model/AggregateDefinition.java?rev=910970&r1=910969&r2=910970&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/model/AggregateDefinition.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/model/AggregateDefinition.java Wed Feb 17 13:44:42 2010
@@ -34,6 +34,7 @@
import org.apache.camel.model.language.ExpressionDefinition;
import org.apache.camel.processor.UnitOfWorkProcessor;
import org.apache.camel.processor.aggregate.AggregateProcessor;
+import org.apache.camel.processor.aggregate.AggregationRepository;
import org.apache.camel.processor.aggregate.AggregationStrategy;
import org.apache.camel.processor.aggregate.GroupedExchangeAggregationStrategy;
import org.apache.camel.spi.RouteContext;
@@ -58,10 +59,14 @@
private AggregationStrategy aggregationStrategy;
@XmlTransient
private ExecutorService executorService;
+ @XmlTransient
+ private AggregationRepository aggregationRepository;
@XmlAttribute(required = false)
private Boolean parallelProcessing;
@XmlAttribute(required = false)
private String executorServiceRef;
+ @XmlAttribute(required = false)
+ private String aggregationRepositoryRef;
@XmlAttribute(required = true)
private String strategyRef;
@XmlAttribute(required = false)
@@ -129,7 +134,8 @@
this.setExpression(clause);
return clause;
}
-
+
+ @SuppressWarnings("unchecked")
protected AggregateProcessor createAggregator(RouteContext routeContext) throws Exception {
Processor processor = routeContext.createProcessor(this);
// wrap the aggregated route in a unit of work processor
@@ -145,6 +151,10 @@
if (isParallelProcessing() != null) {
answer.setParallelProcessing(isParallelProcessing());
}
+ AggregationRepository<Object> repository = createAggregationRepository(routeContext);
+ if (repository != null) {
+ answer.setAggregationRepository(repository);
+ }
if (getCompletionPredicate() != null) {
Predicate predicate = getCompletionPredicate().createPredicate(routeContext);
@@ -187,6 +197,14 @@
return strategy;
}
+ private AggregationRepository createAggregationRepository(RouteContext routeContext) {
+ AggregationRepository repository = getAggregationRepository();
+ if (repository == null && aggregationRepositoryRef != null) {
+ repository = routeContext.lookup(aggregationRepositoryRef, AggregationRepository.class);
+ }
+ return repository;
+ }
+
private ExecutorService createExecutorService(RouteContext routeContext) {
if (executorService == null && executorServiceRef != null) {
executorService = routeContext.lookup(executorServiceRef, ExecutorService.class);
@@ -306,6 +324,22 @@
this.closeCorrelationKeyOnCompletion = closeCorrelationKeyOnCompletion;
}
+ public AggregationRepository getAggregationRepository() {
+ return aggregationRepository;
+ }
+
+ public void setAggregationRepository(AggregationRepository aggregationRepository) {
+ this.aggregationRepository = aggregationRepository;
+ }
+
+ public String getAggregationRepositoryRef() {
+ return aggregationRepositoryRef;
+ }
+
+ public void setAggregationRepositoryRef(String aggregationRepositoryRef) {
+ this.aggregationRepositoryRef = aggregationRepositoryRef;
+ }
+
// Fluent API
//-------------------------------------------------------------------------
@@ -404,6 +438,32 @@
}
/**
+ * Sets the custom aggregate repository to use.
+ * <p/>
+ * Will by default use {@link org.apache.camel.processor.aggregate.MemoryAggregationRepository}
+ *
+ * @param aggregationRepository the aggregate repository to use
+ * @return the builder
+ */
+ public AggregateDefinition aggregationRepository(AggregationRepository aggregationRepository) {
+ setAggregationRepository(aggregationRepository);
+ return this;
+ }
+
+ /**
+ * Sets the custom aggregate repository to use
+ * <p/>
+ * Will by default use {@link org.apache.camel.processor.aggregate.MemoryAggregationRepository}
+ *
+ * @param aggregationRepositoryRef reference to the repository to lookup in the registry
+ * @return the builder
+ */
+ public AggregateDefinition aggregationRepositoryRef(String aggregationRepositoryRef) {
+ setAggregationRepositoryRef(aggregationRepositoryRef);
+ return this;
+ }
+
+ /**
* Enables grouped exchanges, so the aggregator will group all aggregated exchanges into a single
* combined Exchange holding all the aggregated exchanges in a {@link java.util.List} as a exchange
* property with the key {@link org.apache.camel.Exchange#GROUPED_EXCHANGE}.
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java?rev=910970&r1=910969&r2=910970&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java Wed Feb 17 13:44:42 2010
@@ -22,6 +22,7 @@
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.camel.CamelExchangeException;
import org.apache.camel.Exchange;
@@ -85,7 +86,7 @@
private long completionTimeout;
private int completionSize;
private boolean completionFromBatchConsumer;
- private int batchConsumerCounter;
+ private AtomicInteger batchConsumerCounter = new AtomicInteger();
public AggregateProcessor(Processor processor, Expression correlationExpression, AggregationStrategy aggregationStrategy) {
ObjectHelper.notNull(processor, "processor");
@@ -141,11 +142,23 @@
doAggregation(key, exchange);
}
+ /**
+ * Aggregates the exchange with the given correlation key
+ * <p/>
+ * This method <b>must</b> be run synchronized as we cannot aggregate the same correlation key
+ * in parallel.
+ *
+ * @param key the correlation key
+ * @param exchange the exchange
+ * @return the aggregated exchange
+ */
private synchronized Exchange doAggregation(Object key, Exchange exchange) {
- // TODO: lock this based on keys so we can run in parallel groups
+ // when memory based then its fast using synchronized, but if the aggregation repository is IO
+ // bound such as JPA etc then concurrent aggregation per correlation key could
+ // improve performance as we can run aggregation repository get/add in parallel
if (LOG.isTraceEnabled()) {
- LOG.trace("+++ start +++ onAggregation for key " + key);
+ LOG.trace("onAggregation +++ start +++ with correlation key: " + key);
}
Exchange answer;
@@ -193,7 +206,7 @@
}
if (LOG.isTraceEnabled()) {
- LOG.trace("+++ end +++ onAggregation for key " + key + " with size " + size);
+ LOG.trace("onAggregation +++ end +++ with correlation key: " + key);
}
return answer;
@@ -224,11 +237,11 @@
}
if (isCompletionFromBatchConsumer()) {
- batchConsumerCounter++;
+ batchConsumerCounter.incrementAndGet();
int size = exchange.getProperty(Exchange.BATCH_SIZE, 0, Integer.class);
- if (size > 0 && batchConsumerCounter >= size) {
- // batch consumer is complete
- batchConsumerCounter = 0;
+ if (size > 0 && batchConsumerCounter.intValue() >= size) {
+ // batch consumer is complete then reset the counter
+ batchConsumerCounter.set(0);
return true;
}
}
@@ -360,10 +373,18 @@
this.parallelProcessing = parallelProcessing;
}
+ public AggregationRepository<Object> getAggregationRepository() {
+ return aggregationRepository;
+ }
+
+ public void setAggregationRepository(AggregationRepository<Object> aggregationRepository) {
+ this.aggregationRepository = aggregationRepository;
+ }
+
/**
* Background tasks that looks for aggregated exchanges which is triggered by completion timeouts.
*/
- private class AggregationTimeoutMap extends DefaultTimeoutMap<Object, Exchange> {
+ private final class AggregationTimeoutMap extends DefaultTimeoutMap<Object, Exchange> {
private AggregationTimeoutMap(ScheduledExecutorService executor, long requestMapPollTimeMillis) {
super(executor, requestMapPollTimeMillis);
Modified: camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/ClosedCorrelationKeyException.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/ClosedCorrelationKeyException.java?rev=910970&r1=910969&r2=910970&view=diff
==============================================================================
--- camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/ClosedCorrelationKeyException.java (original)
+++ camel/trunk/camel-core/src/main/java/org/apache/camel/processor/aggregate/ClosedCorrelationKeyException.java Wed Feb 17 13:44:42 2010
@@ -27,7 +27,7 @@
public class ClosedCorrelationKeyException extends CamelExchangeException {
// the correlation key is not expected to be serialized
- private transient Object correlationKey;
+ private final transient Object correlationKey;
public ClosedCorrelationKeyException(Object correlationKey, Exchange exchange) {
super("The correlation key [" + correlationKey + "] has been closed", exchange);
Added: camel/trunk/camel-core/src/test/java/org/apache/camel/processor/aggregator/AggregateConcurrentPerCorrelationKeyTest.java
URL: http://svn.apache.org/viewvc/camel/trunk/camel-core/src/test/java/org/apache/camel/processor/aggregator/AggregateConcurrentPerCorrelationKeyTest.java?rev=910970&view=auto
==============================================================================
--- camel/trunk/camel-core/src/test/java/org/apache/camel/processor/aggregator/AggregateConcurrentPerCorrelationKeyTest.java (added)
+++ camel/trunk/camel-core/src/test/java/org/apache/camel/processor/aggregator/AggregateConcurrentPerCorrelationKeyTest.java Wed Feb 17 13:44:42 2010
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor.aggregator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.component.mock.MockEndpoint;
+import org.apache.camel.processor.BodyInAggregatingStrategy;
+
+/**
+ * @version $Revision$
+ */
+public class AggregateConcurrentPerCorrelationKeyTest extends ContextTestSupport {
+
+ private final int size = 200;
+ private final String uri = "direct:start";
+
+ public void testAggregateConcurrentPerCorrelationKey() throws Exception {
+ ExecutorService service = Executors.newFixedThreadPool(20);
+ List<Callable<Object>> tasks = new ArrayList<Callable<Object>>();
+ for (int i = 0; i < size; i++) {
+ final int id = i % 5;
+ final int count = i;
+ tasks.add(new Callable<Object>() {
+ public Object call() throws Exception {
+ template.sendBodyAndHeader(uri, "" + count, "id", id);
+ return null;
+ }
+ });
+ }
+
+ MockEndpoint mock = getMockEndpoint("mock:result");
+ mock.expectedMessageCount(5);
+
+ // submit all tasks
+ service.invokeAll(tasks);
+
+ assertMockEndpointsSatisfied();
+ }
+
+ @Override
+ protected RouteBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ from("direct:start")
+ .aggregate(header("id"), new BodyInAggregatingStrategy()).completionSize(40)
+ .to("mock:result");
+ }
+ };
+ }
+}
Propchange: camel/trunk/camel-core/src/test/java/org/apache/camel/processor/aggregator/AggregateConcurrentPerCorrelationKeyTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: camel/trunk/camel-core/src/test/java/org/apache/camel/processor/aggregator/AggregateConcurrentPerCorrelationKeyTest.java
------------------------------------------------------------------------------
svn:keywords = Rev Date