You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2024/06/28 12:26:50 UTC
(camel) branch main updated: CAMEL-16829: camel-core - Stuck processing with nested parallel split… (#14676)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 6324bc0a6da CAMEL-16829: camel-core - Stuck processing with nested parallel split… (#14676)
6324bc0a6da is described below
commit 6324bc0a6dae05650b851a9524407c5382d3ff6e
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Fri Jun 28 14:26:45 2024 +0200
CAMEL-16829: camel-core - Stuck processing with nested parallel split… (#14676)
* CAMEL-16829: camel-core - Stuck processing with nested parallel splits and custom thread pool
---
.../apache/camel/processor/MulticastProcessor.java | 27 +++++-
.../processor/aggregate/AggregateProcessor.java | 4 +-
.../AggregateParallelThreadPoolAbortTest.java | 92 +++++++++++++++++++++
.../AggregateParallelThreadPoolCallerRunsTest.java | 95 ++++++++++++++++++++++
.../SplitParallelThreadPoolAbortTest.java | 90 ++++++++++++++++++++
.../SplitParallelThreadPoolCallerRunsTest.java | 89 ++++++++++++++++++++
.../throttling/ThrottlingExceptionRoutePolicy.java | 4 +-
7 files changed, 394 insertions(+), 7 deletions(-)
diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java
index 3aa6ea63069..67efcdd56c1 100644
--- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java
+++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/MulticastProcessor.java
@@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@@ -75,6 +76,7 @@ import org.apache.camel.util.CastUtils;
import org.apache.camel.util.IOHelper;
import org.apache.camel.util.StopWatch;
import org.apache.camel.util.concurrent.AsyncCompletionService;
+import org.apache.camel.util.concurrent.Rejectable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -347,7 +349,11 @@ public class MulticastProcessor extends AsyncProcessorSupport
? new MulticastTransactedTask(exchange, pairs, callback, size)
: new MulticastReactiveTask(exchange, pairs, callback, size);
if (isParallelProcessing()) {
- executorService.submit(() -> reactiveExecutor.schedule(state));
+ try {
+ executorService.submit(() -> reactiveExecutor.scheduleSync(state));
+ } catch (RejectedExecutionException e) {
+ state.reject();
+ }
} else {
if (exchange.isTransacted()) {
reactiveExecutor.scheduleQueue(state);
@@ -362,10 +368,16 @@ public class MulticastProcessor extends AsyncProcessorSupport
return false;
}
- protected void schedule(Runnable runnable) {
+ protected void schedule(final Runnable runnable) {
if (isParallelProcessing()) {
Runnable task = prepareParallelTask(runnable);
- executorService.submit(() -> reactiveExecutor.schedule(task));
+ try {
+ executorService.submit(() -> reactiveExecutor.scheduleSync(task));
+ } catch (RejectedExecutionException e) {
+ if (runnable instanceof Rejectable rej) {
+ rej.reject();
+ }
+ }
} else {
reactiveExecutor.schedule(runnable);
}
@@ -402,7 +414,7 @@ public class MulticastProcessor extends AsyncProcessorSupport
return answer;
}
- protected abstract class MulticastTask implements Runnable {
+ protected abstract class MulticastTask implements Runnable, Rejectable {
final Exchange original;
final Iterable<ProcessorExchangePair> pairs;
@@ -527,6 +539,13 @@ public class MulticastProcessor extends AsyncProcessorSupport
MulticastProcessor.this.doDone(original, exchange, pairs, callback, false, forceExhaust);
}
}
+
+ @Override
+ public void reject() {
+ original.setException(new RejectedExecutionException("Task rejected executing from ExecutorService"));
+ // and do the done work
+ doDone(null, false);
+ }
}
/**
diff --git a/core/camel-core-processor/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java b/core/camel-core-processor/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java
index 2c75f512b54..c2f0ebcc45f 100644
--- a/core/camel-core-processor/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java
+++ b/core/camel-core-processor/src/main/java/org/apache/camel/processor/aggregate/AggregateProcessor.java
@@ -897,8 +897,8 @@ public class AggregateProcessor extends AsyncProcessorSupport
LOG.trace("Processing aggregated exchange: {} complete.", exchange);
}
});
- // the call to schedule is needed to ensure in-order processing of the aggregates
- reactiveExecutor.schedule(task);
+ // execute the task using this thread sync (similar to multicast eip in parallel mode)
+ reactiveExecutor.scheduleSync(task);
});
}
diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/AggregateParallelThreadPoolAbortTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/AggregateParallelThreadPoolAbortTest.java
new file mode 100644
index 00000000000..94063f45315
--- /dev/null
+++ b/core/camel-core/src/test/java/org/apache/camel/processor/AggregateParallelThreadPoolAbortTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.builder.ThreadPoolBuilder;
+import org.apache.camel.util.concurrent.ThreadPoolRejectedPolicy;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.parallel.Isolated;
+
+/**
+ * Tests that the EIP does not hang-threads due to thread-pools being exhausted and rejects new tasks
+ */
+@Isolated
+@Timeout(30)
+public class AggregateParallelThreadPoolAbortTest extends ContextTestSupport {
+
+ @Test
+ public void testAggregateParallel() throws Exception {
+ AtomicInteger ok = new AtomicInteger();
+ AtomicInteger fail = new AtomicInteger();
+
+ Runnable r = () -> {
+ try {
+ template.sendBody("direct:start", "Body");
+ ok.incrementAndGet();
+ } catch (Exception e) {
+ fail.incrementAndGet();
+ }
+ };
+ var pool = Executors.newFixedThreadPool(10);
+ for (int i = 0; i < 10; i++) {
+ pool.submit(r);
+ }
+
+ Awaitility.await().untilAsserted(() -> {
+ Assertions.assertTrue(ok.get() > 0, "Some should ok");
+ Assertions.assertTrue(fail.get() > 0, "Some should fail");
+ });
+
+ log.info("Errors: {}", fail.get());
+ log.info("OK: {}", ok.get());
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ final var executorService = new ThreadPoolBuilder(getContext())
+ .poolSize(1)
+ .maxPoolSize(1)
+ .maxQueueSize(0)
+ .rejectedPolicy(ThreadPoolRejectedPolicy.Abort)
+ .build("inner");
+
+ from("direct:start")
+ .to("direct:inner?synchronous=true");
+
+ from("direct:inner")
+ .aggregate(constant(true), new BodyInAggregatingStrategy()).executorService(executorService)
+ .completionSize(2)
+ // force delay so threads are not free in the pool causing tasks to be rejected
+ .delay(1000)
+ .log("${body}");
+ }
+ };
+ }
+
+}
diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/AggregateParallelThreadPoolCallerRunsTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/AggregateParallelThreadPoolCallerRunsTest.java
new file mode 100644
index 00000000000..e1f93215f77
--- /dev/null
+++ b/core/camel-core/src/test/java/org/apache/camel/processor/AggregateParallelThreadPoolCallerRunsTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.builder.ThreadPoolBuilder;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.parallel.Isolated;
+
+/**
+ * Tests that the EIP does not hang-threads due to thread-pools being exhausted and uses caller runs to use current
+ * thread to process the task
+ */
+@Isolated
+@Timeout(30)
+public class AggregateParallelThreadPoolCallerRunsTest extends ContextTestSupport {
+
+ @Test
+ public void testAggregateParallel() throws Exception {
+ getMockEndpoint("mock:result").expectedMessageCount(5);
+
+ AtomicInteger ok = new AtomicInteger();
+ AtomicInteger fail = new AtomicInteger();
+
+ Runnable r = () -> {
+ try {
+ template.sendBody("direct:start", "Body");
+ ok.incrementAndGet();
+ } catch (Exception e) {
+ fail.incrementAndGet();
+ }
+ };
+ var pool = Executors.newFixedThreadPool(10);
+ for (int i = 0; i < 10; i++) {
+ pool.submit(r);
+ }
+
+ Awaitility.await().untilAsserted(() -> {
+ Assertions.assertEquals(10, ok.get(), "All should be ok");
+ });
+
+ assertMockEndpointsSatisfied();
+
+ log.info("Errors: {}", fail.get());
+ log.info("OK: {}", ok.get());
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ final var executorService = new ThreadPoolBuilder(getContext())
+ .poolSize(1)
+ .maxPoolSize(1)
+ .maxQueueSize(0)
+ .build("inner");
+
+ from("direct:start")
+ .to("direct:inner?synchronous=true");
+
+ from("direct:inner")
+ .aggregate(constant(true), new BodyInAggregatingStrategy()).executorService(executorService)
+ .completionSize(2)
+ // force delay so threads are not free in the pool causing tasks to be rejected
+ .delay(1000)
+ .log("${body}")
+ .to("mock:result");
+ }
+ };
+ }
+
+}
diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitParallelThreadPoolAbortTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitParallelThreadPoolAbortTest.java
new file mode 100644
index 00000000000..71815293e6b
--- /dev/null
+++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitParallelThreadPoolAbortTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.builder.ThreadPoolBuilder;
+import org.apache.camel.util.concurrent.ThreadPoolRejectedPolicy;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.parallel.Isolated;
+
+/**
+ * Tests that the EIP does not hang-threads due to thread-pools being exhausted and rejects new tasks
+ */
+@Isolated
+@Timeout(30)
+public class SplitParallelThreadPoolAbortTest extends ContextTestSupport {
+
+ @Test
+ public void testSplitParallel() throws Exception {
+ AtomicInteger ok = new AtomicInteger();
+ AtomicInteger fail = new AtomicInteger();
+
+ Runnable r = () -> {
+ try {
+ template.sendBody("direct:start", List.of(List.of("0-0", "0-1"), List.of("1-0", "1-1")));
+ ok.incrementAndGet();
+ } catch (Exception e) {
+ fail.incrementAndGet();
+ }
+ };
+ var pool = Executors.newFixedThreadPool(10);
+ for (int i = 0; i < 10; i++) {
+ pool.submit(r);
+ }
+
+ Awaitility.await().untilAsserted(() -> {
+ Assertions.assertEquals(10, fail.get(), "All should fail");
+ });
+
+ log.info("Errors: {}", fail.get());
+ log.info("OK: {}", ok.get());
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ final var executorService = new ThreadPoolBuilder(getContext())
+ .poolSize(1)
+ .maxPoolSize(1)
+ .maxQueueSize(0)
+ .rejectedPolicy(ThreadPoolRejectedPolicy.Abort)
+ .build("inner");
+
+ from("direct:start")
+ .split(body()).parallelProcessing()
+ .to("direct:inner?synchronous=true");
+
+ from("direct:inner")
+ .split(body()).executorService(executorService)
+ .log("${body}");
+ }
+ };
+ }
+
+}
diff --git a/core/camel-core/src/test/java/org/apache/camel/processor/SplitParallelThreadPoolCallerRunsTest.java b/core/camel-core/src/test/java/org/apache/camel/processor/SplitParallelThreadPoolCallerRunsTest.java
new file mode 100644
index 00000000000..aa49ee25f34
--- /dev/null
+++ b/core/camel-core/src/test/java/org/apache/camel/processor/SplitParallelThreadPoolCallerRunsTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.processor;
+
+import java.util.List;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.camel.ContextTestSupport;
+import org.apache.camel.RoutesBuilder;
+import org.apache.camel.builder.RouteBuilder;
+import org.apache.camel.builder.ThreadPoolBuilder;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.parallel.Isolated;
+
+/**
+ * Tests that the EIP does not hang-threads due to thread-pools being exhausted and uses caller runs to use current
+ * thread to process the task
+ */
+@Isolated
+@Timeout(30)
+public class SplitParallelThreadPoolCallerRunsTest extends ContextTestSupport {
+
+ @Test
+ public void testSplitParallel() throws Exception {
+ AtomicInteger ok = new AtomicInteger();
+ AtomicInteger fail = new AtomicInteger();
+
+ Runnable r = () -> {
+ try {
+ template.sendBody("direct:start", List.of(List.of("0-0", "0-1"), List.of("1-0", "1-1")));
+ ok.incrementAndGet();
+ } catch (Exception e) {
+ fail.incrementAndGet();
+ }
+ };
+ var pool = Executors.newFixedThreadPool(10);
+ for (int i = 0; i < 10; i++) {
+ pool.submit(r);
+ }
+
+ Awaitility.await().untilAsserted(() -> {
+ Assertions.assertEquals(10, ok.get(), "All should be okay");
+ });
+
+ log.info("Errors: {}", fail.get());
+ log.info("OK: {}", ok.get());
+ }
+
+ @Override
+ protected RoutesBuilder createRouteBuilder() throws Exception {
+ return new RouteBuilder() {
+ @Override
+ public void configure() throws Exception {
+ final var executorService = new ThreadPoolBuilder(getContext())
+ .poolSize(1)
+ .maxPoolSize(1)
+ .maxQueueSize(0)
+ .build("inner");
+
+ from("direct:start")
+ .split(body()).parallelProcessing()
+ .to("direct:inner?synchronous=true");
+
+ from("direct:inner")
+ .split(body()).executorService(executorService)
+ .log("${body}");
+ }
+ };
+ }
+
+}
diff --git a/core/camel-support/src/main/java/org/apache/camel/throttling/ThrottlingExceptionRoutePolicy.java b/core/camel-support/src/main/java/org/apache/camel/throttling/ThrottlingExceptionRoutePolicy.java
index bb4c01fe6d7..0595eed5bc4 100644
--- a/core/camel-support/src/main/java/org/apache/camel/throttling/ThrottlingExceptionRoutePolicy.java
+++ b/core/camel-support/src/main/java/org/apache/camel/throttling/ThrottlingExceptionRoutePolicy.java
@@ -394,7 +394,9 @@ public class ThrottlingExceptionRoutePolicy extends RoutePolicySupport implement
@Override
public void run() {
- halfOpenTimer.cancel();
+ if (halfOpenTimer != null) {
+ halfOpenTimer.cancel();
+ }
calculateState(route);
}
}