You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by da...@apache.org on 2023/12/16 16:38:25 UTC
(camel) branch rc created (now 10ec8dbe65d)
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a change to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
at 10ec8dbe65d CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
This branch includes the following new commits:
new 6311c56ae01 CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
new 606d52fa236 CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
new 070e68f937e CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
new 62a36d0361f CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
new 7caafe2cc6f CAMEL-20242: camel-main - Health check should not be as verbose by default.
new 2cb76b73aee CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
new 10ec8dbe65d CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(camel) 04/07: CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 62a36d0361f380713f9123891734e29d179caf3f
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Dec 16 15:13:04 2023 +0100
CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
---
.../src/main/java/org/apache/camel/spi/RouteController.java | 5 +++--
.../main/java/org/apache/camel/spi/SupervisingRouteController.java | 4 ++--
.../java/org/apache/camel/impl/engine/InternalRouteController.java | 2 --
.../org/apache/camel/impl/health/RouteControllerHealthCheck.java | 1 -
4 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java b/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java
index d538eb77d97..7aace75c0f6 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java
@@ -100,8 +100,9 @@ public interface RouteController extends CamelContextAware, StaticService {
boolean isStartingRoutes();
/**
- * Indicates if the route controller has routes that are unhealthy such as they have not yet been successfully started,
- * and if being supervised then the route can either be pending restarts or failed all restart attempts and are exhausted.
+ * Indicates if the route controller has routes that are unhealthy such as they have not yet been successfully
+ * started, and if being supervised then the route can either be pending restarts or failed all restart attempts and
+ * are exhausted.
*/
boolean isUnhealthyRoutes();
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java b/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
index a2018986e1b..f80e8248bda 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
@@ -174,8 +174,8 @@ public interface SupervisingRouteController extends RouteController {
Throwable getRestartException(String routeId);
/**
- * Whether the route controller is currently starting routes for the first time.
- * This only reports on the first time start phase.
+ * Whether the route controller is currently starting routes for the first time. This only reports on the first time
+ * start phase.
*/
boolean isStartingRoutes();
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java
index e07a99760cd..a3e49591e8e 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java
@@ -27,8 +27,6 @@ import org.apache.camel.spi.RouteController;
import org.apache.camel.spi.RouteError;
import org.apache.camel.spi.SupervisingRouteController;
-import static org.apache.camel.support.service.ServiceHelper.isStarted;
-
/**
* Internal {@link RouteController} used internally by {@link AbstractCamelContext}.
*/
diff --git a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
index 06b076e44f1..16b9b08caed 100644
--- a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
+++ b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
@@ -21,7 +21,6 @@ import java.util.Map;
import org.apache.camel.Ordered;
import org.apache.camel.health.HealthCheckResultBuilder;
import org.apache.camel.spi.RouteController;
-import org.apache.camel.support.service.ServiceHelper;
/**
* Readiness {@link org.apache.camel.health.HealthCheck} for route controller.
(camel) 06/07: CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 2cb76b73aeec025b1472d5b5c291f8c6091c7629
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Dec 16 16:56:04 2023 +0100
CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
---
.../camel/core/xml/AbstractCamelContextFactoryBean.java | 8 ++++++--
.../camel/core/xml/CamelRouteControllerDefinition.java | 17 +++++++++++++++++
.../camel/impl/health/RouteControllerHealthCheck.java | 6 ++++--
.../modules/ROOT/pages/route-controller.adoc | 2 ++
4 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/AbstractCamelContextFactoryBean.java b/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/AbstractCamelContextFactoryBean.java
index 9e4f9db6442..316ccf72302 100644
--- a/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/AbstractCamelContextFactoryBean.java
+++ b/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/AbstractCamelContextFactoryBean.java
@@ -880,11 +880,15 @@ public abstract class AbstractCamelContextFactoryBean<T extends ModelCamelContex
src.setBackOffMultiplier(backOffMultiplier);
}
Boolean unhealthyOnExhausted = CamelContextHelper.parseBoolean(getContext(), rc.getUnhealthyOnExhausted());
- if (unhealthyOnExhausted != null && unhealthyOnExhausted) {
+ if (src != null && unhealthyOnExhausted != null && unhealthyOnExhausted) {
src.setUnhealthyOnExhausted(unhealthyOnExhausted);
}
+ Boolean unhealthyOnRestarting = CamelContextHelper.parseBoolean(getContext(), rc.getUnhealthyOnRestarting());
+ if (src != null && unhealthyOnRestarting != null && unhealthyOnRestarting) {
+ src.setUnhealthyOnRestarting(unhealthyOnRestarting);
+ }
LoggingLevel loggingLevel = CamelContextHelper.parse(getContext(), LoggingLevel.class, rc.getLoggingLevel());
- if (loggingLevel != null) {
+ if (src != null && loggingLevel != null) {
src.setLoggingLevel(loggingLevel);
}
}
diff --git a/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/CamelRouteControllerDefinition.java b/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/CamelRouteControllerDefinition.java
index 3e3ec752627..fa09fc2ccbf 100644
--- a/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/CamelRouteControllerDefinition.java
+++ b/core/camel-core-xml/src/main/java/org/apache/camel/core/xml/CamelRouteControllerDefinition.java
@@ -60,6 +60,9 @@ public class CamelRouteControllerDefinition extends IdentifiedType {
@Metadata(defaultValue = "false", javaType = "java.lang.Boolean")
private String unhealthyOnExhausted;
@XmlAttribute
+ @Metadata(defaultValue = "false", javaType = "java.lang.Boolean")
+ private String unhealthyOnRestarting;
+ @XmlAttribute
@Metadata(javaType = "org.apache.camel.LoggingLevel", defaultValue = "DEBUG", enums = "TRACE,DEBUG,INFO,WARN,ERROR,OFF")
private String loggingLevel;
@@ -209,6 +212,20 @@ public class CamelRouteControllerDefinition extends IdentifiedType {
this.unhealthyOnExhausted = unhealthyOnExhausted;
}
+ public String getUnhealthyOnRestarting() {
+ return unhealthyOnRestarting;
+ }
+
+ /**
+ * Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled
+ * for restarting (backoff).
+ *
+ * Setting this to true allows health checks to know about this and can report the Camel application as DOWN.
+ */
+ public void setUnhealthyOnRestarting(String unhealthyOnRestarting) {
+ this.unhealthyOnRestarting = unhealthyOnRestarting;
+ }
+
public String getLoggingLevel() {
return loggingLevel;
}
diff --git a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
index 00639f256b6..5f23eb6e6a1 100644
--- a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
+++ b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
@@ -76,7 +76,8 @@ public class RouteControllerHealthCheck extends AbstractHealthCheck {
}
}
- private void builderDetails(HealthCheckResultBuilder builder, SupervisingRouteController src, Route route, boolean exhausted) {
+ private void builderDetails(
+ HealthCheckResultBuilder builder, SupervisingRouteController src, Route route, boolean exhausted) {
String routeId = route.getRouteId();
Throwable cause = src.getRestartException(routeId);
if (cause != null) {
@@ -114,7 +115,8 @@ public class RouteControllerHealthCheck extends AbstractHealthCheck {
String msg;
if (exhausted) {
msg = String.format("Restarting route: %s is exhausted after %s attempts due %s."
- + " No more attempts will be made and the route is no longer supervised by this route controller and remains as stopped.", routeId, attempts,
+ + " No more attempts will be made and the route is no longer supervised by this route controller and remains as stopped.",
+ routeId, attempts,
cause.getMessage());
} else {
msg = String.format("Failed restarting route: %s attempt: %s due: %s", routeId, attempts,
diff --git a/docs/user-manual/modules/ROOT/pages/route-controller.adoc b/docs/user-manual/modules/ROOT/pages/route-controller.adoc
index 095a042b1ee..0d3dd197f1c 100644
--- a/docs/user-manual/modules/ROOT/pages/route-controller.adoc
+++ b/docs/user-manual/modules/ROOT/pages/route-controller.adoc
@@ -143,6 +143,8 @@ You can configure the `SupervisingRouteController` using the following options:
| IncludeRoutes | | Pattern for filtering routes to be included as supervised. The pattern is matching on route id, and endpoint uri for the route. Multiple patterns can be separated by comma. For example to include all kafka routes, you can say kafka:. And to include routes with specific route ids myRoute,myOtherRoute. The pattern supports wildcards and uses the matcher from org.apache.camel.support.PatternHelper#matchPattern.
| ExcludeRoutes | | Pattern for filtering routes to be excluded as supervised. The pattern is matching on route id, and endpoint uri for the route. Multiple patterns can be separated by comma. For example to exclude all JMS routes, you can say jms:. And to exclude routes with specific route ids mySpecialRoute,myOtherSpecialRoute. The pattern supports wildcards and uses the matcher from org.apache.camel.support.PatternHelper#matchPattern.
| ThreadPoolSize | `1` | The number of threads used by the route controller scheduled thread pool that are used for restarting routes. The pool uses 1 thread by default, but you can increase this to allow the controller to concurrently attempt to restart multiple routes in case more than one route has problems starting.
+| UnhealthyOnExhausted | `false` | Whether to mark the route as unhealthy (down) when all restarting attempts (backoff) have failed and the route is not successfully started and the route manager is giving up. Setting this to true allows health checks to know about this and can report the Camel application as DOWN.
+| routeControllerUnhealthyOnRestarting | `false` | Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled for restarting (backoff). Setting this to true allows health checks to know about this and can report the Camel application as DOWN.
|=======================================================================
=== Filtering routes to fail fast
(camel) 05/07: CAMEL-20242: camel-main - Health check should not be as verbose by default.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 7caafe2cc6f0601c048fb10596a86b279f5acf6d
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Dec 16 16:47:22 2023 +0100
CAMEL-20242: camel-main - Health check should not be as verbose by default.
---
.../platform/http/main/MainHttpServer.java | 55 ++++++++++-------
.../camel/impl/console/HealthDevConsole.java | 19 +++---
.../impl/health/RouteControllerHealthCheck.java | 70 ++++++++++++++++++++++
3 files changed, 115 insertions(+), 29 deletions(-)
diff --git a/components/camel-platform-http-main/src/main/java/org/apache/camel/component/platform/http/main/MainHttpServer.java b/components/camel-platform-http-main/src/main/java/org/apache/camel/component/platform/http/main/MainHttpServer.java
index 32e1ff68f8e..840eb824bd0 100644
--- a/components/camel-platform-http-main/src/main/java/org/apache/camel/component/platform/http/main/MainHttpServer.java
+++ b/components/camel-platform-http-main/src/main/java/org/apache/camel/component/platform/http/main/MainHttpServer.java
@@ -323,28 +323,30 @@ public class MainHttpServer extends ServiceSupport implements CamelContextAware,
public void handle(RoutingContext ctx) {
ctx.response().putHeader("content-type", "application/json");
+ HealthCheckRegistry registry = HealthCheckRegistry.get(camelContext);
+ String level = ctx.request().getParam("exposureLevel");
+ if (level == null) {
+ level = registry.getExposureLevel();
+ }
+ String includeStackTrace = ctx.request().getParam("stackTrace");
+ String includeData = ctx.request().getParam("data");
+
boolean all = ctx.currentRoute() == health;
boolean liv = ctx.currentRoute() == live;
boolean rdy = ctx.currentRoute() == ready;
Collection<HealthCheck.Result> res;
if (all) {
- res = HealthCheckHelper.invoke(camelContext);
+ res = HealthCheckHelper.invoke(camelContext, level);
} else if (liv) {
- res = HealthCheckHelper.invokeLiveness(camelContext);
+ res = HealthCheckHelper.invokeLiveness(camelContext, level);
} else {
- res = HealthCheckHelper.invokeReadiness(camelContext);
+ res = HealthCheckHelper.invokeReadiness(camelContext, level);
}
StringBuilder sb = new StringBuilder();
sb.append("{\n");
- HealthCheckRegistry registry = HealthCheckRegistry.get(camelContext);
- String level = ctx.request().getParam("exposureLevel");
- if (level == null) {
- level = registry.getExposureLevel();
- }
-
// are we UP
boolean up = HealthCheckHelper.isResultsUp(res, rdy);
@@ -354,12 +356,12 @@ public class MainHttpServer extends ServiceSupport implements CamelContextAware,
} else if ("full".equals(level)) {
// include all details
List<HealthCheck.Result> list = new ArrayList<>(res);
- healthCheckDetails(sb, list, up);
+ healthCheckDetails(sb, list, up, level, includeStackTrace, includeData);
} else {
// include only DOWN details
List<HealthCheck.Result> downs = res.stream().filter(r -> r.getState().equals(HealthCheck.State.DOWN))
.collect(Collectors.toList());
- healthCheckDetails(sb, downs, up);
+ healthCheckDetails(sb, downs, up, level, includeStackTrace, includeData);
}
sb.append("}\n");
@@ -391,7 +393,9 @@ public class MainHttpServer extends ServiceSupport implements CamelContextAware,
}
}
- private static void healthCheckDetails(StringBuilder sb, List<HealthCheck.Result> checks, boolean up) {
+ private static void healthCheckDetails(
+ StringBuilder sb, List<HealthCheck.Result> checks, boolean up, String level, String includeStackTrace,
+ String includeData) {
healthCheckStatus(sb, up);
if (!checks.isEmpty()) {
@@ -400,7 +404,7 @@ public class MainHttpServer extends ServiceSupport implements CamelContextAware,
for (int i = 0; i < checks.size(); i++) {
HealthCheck.Result d = checks.get(i);
sb.append(" {\n");
- reportHealthCheck(sb, d);
+ reportHealthCheck(sb, d, level, includeStackTrace, includeData);
if (i < checks.size() - 1) {
sb.append(" },\n");
} else {
@@ -411,20 +415,29 @@ public class MainHttpServer extends ServiceSupport implements CamelContextAware,
}
}
- private static void reportHealthCheck(StringBuilder sb, HealthCheck.Result d) {
+ private static void reportHealthCheck(
+ StringBuilder sb, HealthCheck.Result d, String level, String includeStackTrace, String includeData) {
sb.append(" \"name\": \"").append(d.getCheck().getId()).append("\",\n");
- sb.append(" \"status\": \"").append(d.getState()).append("\",\n");
- if (d.getError().isPresent()) {
+ sb.append(" \"status\": \"").append(d.getState()).append("\"");
+ if (("full".equals(level) || "true".equals(includeStackTrace)) && d.getError().isPresent()) {
+ // include error message in full exposure
+ sb.append(",\n");
String msg = allCausedByErrorMessages(d.getError().get());
sb.append(" \"error-message\": \"").append(msg)
- .append("\",\n");
- sb.append(" \"error-stacktrace\": \"").append(errorStackTrace(d.getError().get()))
- .append("\",\n");
+ .append("\"");
+ if ("true".equals(includeStackTrace)) {
+ sb.append(",\n");
+ sb.append(" \"error-stacktrace\": \"").append(errorStackTrace(d.getError().get()))
+ .append("\"");
+ }
}
if (d.getMessage().isPresent()) {
- sb.append(" \"message\": \"").append(d.getMessage().get()).append("\",\n");
+ sb.append(",\n");
+ sb.append(" \"message\": \"").append(d.getMessage().get()).append("\"");
}
- if (d.getDetails() != null && !d.getDetails().isEmpty()) {
+ // only include data if was enabled
+ if (("true".equals(includeData)) && d.getDetails() != null && !d.getDetails().isEmpty()) {
+ sb.append(",\n");
// lets use sorted keys
Iterator<String> it = new TreeSet<>(d.getDetails().keySet()).iterator();
sb.append(" \"data\": {\n");
diff --git a/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java b/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java
index 175849de7c2..85e2ab46327 100644
--- a/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java
+++ b/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java
@@ -53,17 +53,20 @@ public class HealthDevConsole extends AbstractDevConsole {
sb.append(String.format("\n %s: %s", res.getCheck().getId(), res.getState()));
} else {
if (res.getMessage().isPresent()) {
- sb.append(String.format("\n %s: %s (%s)", res.getCheck().getId(), res.getState(), res.getMessage()));
+ sb.append(String.format("\n %s: %s (%s)", res.getCheck().getId(), res.getState(), res.getMessage().get()));
} else {
sb.append(String.format("\n %s: %s", res.getCheck().getId(), res.getState()));
}
- Throwable cause = res.getError().orElse(null);
- if (cause != null) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- cause.printStackTrace(pw);
- sb.append(pw);
- sb.append("\n\n");
+ if ("full".equals(exposureLevel)) {
+ if (res.getError().isPresent()) {
+ Throwable cause = res.getError().get();
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ cause.printStackTrace(pw);
+ sb.append("\n\n");
+ sb.append(sw);
+ sb.append("\n\n");
+ }
}
}
});
diff --git a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
index 16b9b08caed..00639f256b6 100644
--- a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
+++ b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
@@ -16,11 +16,18 @@
*/
package org.apache.camel.impl.health;
+import java.util.Comparator;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import org.apache.camel.Ordered;
+import org.apache.camel.Route;
import org.apache.camel.health.HealthCheckResultBuilder;
import org.apache.camel.spi.RouteController;
+import org.apache.camel.spi.SupervisingRouteController;
+import org.apache.camel.util.URISupport;
+import org.apache.camel.util.backoff.BackOffTimer;
/**
* Readiness {@link org.apache.camel.health.HealthCheck} for route controller.
@@ -46,6 +53,19 @@ public class RouteControllerHealthCheck extends AbstractHealthCheck {
if (rc != null) {
// should only be up if there are no unhealthy routes
up = !rc.isUnhealthyRoutes();
+ // do we have any details about why we are not up
+ if (!up && rc instanceof SupervisingRouteController src) {
+ Set<Route> routes = new TreeSet<>(Comparator.comparing(Route::getId));
+ routes.addAll(src.getRestartingRoutes());
+ for (Route route : routes) {
+ builderDetails(builder, src, route, false);
+ }
+ routes = new TreeSet<>(Comparator.comparing(Route::getId));
+ routes.addAll(src.getExhaustedRoutes());
+ for (Route route : routes) {
+ builderDetails(builder, src, route, true);
+ }
+ }
}
if (up) {
@@ -56,4 +76,54 @@ public class RouteControllerHealthCheck extends AbstractHealthCheck {
}
}
+ private void builderDetails(HealthCheckResultBuilder builder, SupervisingRouteController src, Route route, boolean exhausted) {
+ String routeId = route.getRouteId();
+ Throwable cause = src.getRestartException(routeId);
+ if (cause != null) {
+ String status = src.getRouteStatus(routeId).name();
+ String uri = route.getEndpoint().getEndpointBaseUri();
+ uri = URISupport.sanitizeUri(uri);
+
+ BackOffTimer.Task state = src.getRestartingRouteState(routeId);
+ long attempts = state != null ? state.getCurrentAttempts() : 0;
+ long elapsed;
+ long last;
+ long next;
+ // we can only track elapsed/time for active supervised routes
+ elapsed = state != null && BackOffTimer.Task.Status.Active == state.getStatus()
+ ? state.getCurrentElapsedTime() : 0;
+ last = state != null && BackOffTimer.Task.Status.Active == state.getStatus()
+ ? state.getLastAttemptTime() : 0;
+ next = state != null && BackOffTimer.Task.Status.Active == state.getStatus()
+ ? state.getNextAttemptTime() : 0;
+
+ String key = "route." + routeId;
+ builder.detail(key + ".id", routeId);
+ builder.detail(key + ".status", status);
+ builder.detail(key + ".phase", exhausted ? "Exhausted" : "Restarting");
+ builder.detail(key + ".uri", uri);
+ builder.detail(key + ".attempts", attempts);
+ builder.detail(key + ".lastAttempt", last);
+ builder.detail(key + ".nextAttempt", next);
+ builder.detail(key + ".elapsed", elapsed);
+ if (cause.getMessage() != null) {
+ builder.detail(key + ".error", cause.getMessage());
+ // only one exception can be stored so lets just store first found
+ if (builder.error() == null) {
+ builder.error(cause);
+ String msg;
+ if (exhausted) {
+ msg = String.format("Restarting route: %s is exhausted after %s attempts due %s."
+ + " No more attempts will be made and the route is no longer supervised by this route controller and remains as stopped.", routeId, attempts,
+ cause.getMessage());
+ } else {
+ msg = String.format("Failed restarting route: %s attempt: %s due: %s", routeId, attempts,
+ cause.getMessage());
+ }
+ builder.message(msg);
+ }
+ }
+ }
+ }
+
}
(camel) 02/07: CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 606d52fa236dc945d71b873a1a293616fa3c48a4
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Dec 16 12:48:12 2023 +0100
CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
---
.../impl/engine/DefaultSupervisingRouteController.java | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
index 8b943f34c20..44b3b6cc6a0 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
@@ -249,7 +249,19 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
@Override
public boolean isStartingRoutes() {
- return startingRoutes;
+ boolean answer = startingRoutes;
+
+ // if we have started the routes first time, but some failed and are scheduled for restart
+ // then we may report as still starting routes if we should be unhealthy on restarting
+ if (!answer && isUnhealthyOnRestarting()) {
+ // mark as still starting routes if we have routes to restart
+ answer = !routeManager.routes.isEmpty();
+ }
+ if (!answer && isUnhealthyOnExhausted()) {
+ // mark as still starting routes if we have exhausted routes that should be unhealthy
+ answer = !routeManager.exhausted.isEmpty();
+ }
+ return answer;
}
@Override
@@ -621,7 +633,8 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
if (!getCamelContext().isRunAllowed()) {
// Camel is shutting down so do not attempt to start route
- logger.info("Restarting route: {} attempt: {} is cancelled due CamelContext is shutting down", r.getId(), attempt);
+ logger.info("Restarting route: {} attempt: {} is cancelled due CamelContext is shutting down",
+ r.getId(), attempt);
return true;
}
(camel) 07/07: CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 10ec8dbe65d001f45272785bf7155ea1307966da
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Dec 16 17:25:51 2023 +0100
CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
---
.../src/main/java/org/apache/camel/impl/console/HealthDevConsole.java | 3 ++-
.../generated/resources/org/apache/camel/core/xml/routeController.json | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java b/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java
index 85e2ab46327..75aca65610a 100644
--- a/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java
+++ b/core/camel-console/src/main/java/org/apache/camel/impl/console/HealthDevConsole.java
@@ -53,7 +53,8 @@ public class HealthDevConsole extends AbstractDevConsole {
sb.append(String.format("\n %s: %s", res.getCheck().getId(), res.getState()));
} else {
if (res.getMessage().isPresent()) {
- sb.append(String.format("\n %s: %s (%s)", res.getCheck().getId(), res.getState(), res.getMessage().get()));
+ sb.append(
+ String.format("\n %s: %s (%s)", res.getCheck().getId(), res.getState(), res.getMessage().get()));
} else {
sb.append(String.format("\n %s: %s", res.getCheck().getId(), res.getState()));
}
diff --git a/core/camel-core-xml/src/generated/resources/org/apache/camel/core/xml/routeController.json b/core/camel-core-xml/src/generated/resources/org/apache/camel/core/xml/routeController.json
index e10475e45b3..241a57d2de1 100644
--- a/core/camel-core-xml/src/generated/resources/org/apache/camel/core/xml/routeController.json
+++ b/core/camel-core-xml/src/generated/resources/org/apache/camel/core/xml/routeController.json
@@ -24,6 +24,7 @@
"backOffMaxAttempts": { "index": 9, "kind": "attribute", "displayName": "Back Off Max Attempts", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "description": "Backoff maximum number of attempts to restart a route that failed to startup. When this threshold has been exceeded then the controller will give up attempting to restart the route, and the route will remain as stopped." },
"backOffMultiplier": { "index": 10, "kind": "attribute", "displayName": "Back Off Multiplier", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "1.0", "description": "Backoff multiplier to use for exponential backoff. This is used to extend the delay between restart attempts." },
"unhealthyOnExhausted": { "index": 11, "kind": "attribute", "displayName": "Unhealthy On Exhausted", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether to mark the route as unhealthy (down) when all restarting attempts (backoff) have failed and the route is not successfully started and the route manager is giving up. Setting this to true allows health checks to [...]
- "loggingLevel": { "index": 12, "kind": "attribute", "displayName": "Logging Level", "required": false, "type": "enum", "javaType": "org.apache.camel.LoggingLevel", "enum": [ "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "OFF" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "DEBUG", "description": "Sets the logging level used for logging route activity (such as starting and stopping routes). The default logging level is DEBUG." }
+ "unhealthyOnRestarting": { "index": 12, "kind": "attribute", "displayName": "Unhealthy On Restarting", "required": false, "type": "boolean", "javaType": "java.lang.Boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled for restarting (backoff). Setting this to true allows health checks to know about this and can report the [...]
+ "loggingLevel": { "index": 13, "kind": "attribute", "displayName": "Logging Level", "required": false, "type": "enum", "javaType": "org.apache.camel.LoggingLevel", "enum": [ "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "OFF" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "DEBUG", "description": "Sets the logging level used for logging route activity (such as starting and stopping routes). The default logging level is DEBUG." }
}
}
(camel) 01/07: CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 6311c56ae018c7557357b57cefa325ae8ea3b96a
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Dec 16 12:04:46 2023 +0100
CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
---
.../camel/spi/SupervisingRouteController.java | 14 +++++
.../engine/DefaultSupervisingRouteController.java | 32 ++++++++++-
.../camel/impl/console/RouteControllerConsole.java | 2 +
.../camel/health-check/route-controller-check | 2 +
.../impl/health/RouteControllerHealthCheck.java | 64 ++++++++++++++++++++++
.../MainConfigurationPropertiesConfigurer.java | 6 ++
.../camel-main-configuration-metadata.json | 1 +
core/camel-main/src/main/docs/main.adoc | 3 +-
.../camel/main/DefaultConfigurationConfigurer.java | 1 +
.../camel/main/DefaultConfigurationProperties.java | 30 ++++++++++
.../org/apache/camel/util/backoff/BackOff.java | 8 ++-
.../commands/action/RouteControllerAction.java | 19 ++++---
12 files changed, 171 insertions(+), 11 deletions(-)
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java b/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
index dabd4edf1c8..4631919cfbc 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
@@ -128,6 +128,18 @@ public interface SupervisingRouteController extends RouteController {
*/
boolean isUnhealthyOnExhausted();
+ boolean isUnhealthyOnRestarting();
+
+ /**
+ * Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled
+ * for restarting (backoff).
+ *
+ * Setting this to true allows health checks to know about this and can report the Camel application as DOWN.
+ *
+ * The default is false.
+ */
+ void setUnhealthyOnRestarting(boolean unhealthyOnRestarting);
+
/**
* Return the list of routes that are currently under restarting by this controller.
*
@@ -161,4 +173,6 @@ public interface SupervisingRouteController extends RouteController {
*/
Throwable getRestartException(String routeId);
+ boolean isStartingRoutes();
+
}
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
index 2f0e72ed0e5..8b943f34c20 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
@@ -75,6 +75,7 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
private final Set<String> nonSupervisedRoutes;
private final RouteManager routeManager;
private volatile CamelContextStartupListener listener;
+ private volatile boolean startingRoutes = true; // state during starting routes on bootstrap
private volatile BackOffTimer timer;
private volatile ScheduledExecutorService executorService;
private volatile BackOff backOff;
@@ -88,6 +89,7 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
private long backOffMaxAttempts;
private double backOffMultiplier = 1.0d;
private boolean unhealthyOnExhausted;
+ private boolean unhealthyOnRestarting;
public DefaultSupervisingRouteController() {
this.lock = new Object();
@@ -182,6 +184,14 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
this.unhealthyOnExhausted = unhealthyOnExhausted;
}
+ public boolean isUnhealthyOnRestarting() {
+ return unhealthyOnRestarting;
+ }
+
+ public void setUnhealthyOnRestarting(boolean unhealthyOnRestarting) {
+ this.unhealthyOnRestarting = unhealthyOnRestarting;
+ }
+
protected BackOff getBackOff(String id) {
// currently all routes use the same backoff
return backOff;
@@ -237,6 +247,11 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
// Route management
// *********************************
+ @Override
+ public boolean isStartingRoutes() {
+ return startingRoutes;
+ }
+
@Override
public void startRoute(String routeId) throws Exception {
final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst();
@@ -457,6 +472,14 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
}
private void startSupervisedRoutes() {
+ try {
+ doStartSupervisedRoutes();
+ } finally {
+ startingRoutes = false;
+ }
+ }
+
+ private void doStartSupervisedRoutes() {
if (!isRunAllowed()) {
return;
}
@@ -482,7 +505,7 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
if (getCamelContext().getStartupSummaryLevel() != StartupSummaryLevel.Off
&& getCamelContext().getStartupSummaryLevel() != StartupSummaryLevel.Oneline) {
- // log after first round of attempts
+ // log after first round of attempts (some routes may be scheduled for restart)
logRouteStartupSummary();
}
}
@@ -595,6 +618,13 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
BackOffTimer.Task task = timer.schedule(backOff, context -> {
final BackOffTimer.Task state = getBackOffContext(r.getId()).orElse(null);
long attempt = state != null ? state.getCurrentAttempts() : 0;
+
+ if (!getCamelContext().isRunAllowed()) {
+ // Camel is shutting down so do not attempt to start route
+ logger.info("Restarting route: {} attempt: {} is cancelled due CamelContext is shutting down", r.getId(), attempt);
+ return true;
+ }
+
try {
logger.info("Restarting route: {} attempt: {}", r.getId(), attempt);
doStartRoute(r, false, rx -> DefaultSupervisingRouteController.super.startRoute(rx.getId()));
diff --git a/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java b/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java
index cacfc0c8f86..c43e1fa1960 100644
--- a/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java
+++ b/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java
@@ -76,6 +76,7 @@ public class RouteControllerConsole extends AbstractDevConsole {
sb.append(String.format("\nBackoff Max Elapsed Time: %d", src.getBackOffMaxElapsedTime()));
sb.append(String.format("\nBackoff Max Attempts: %d", src.getBackOffMaxAttempts()));
sb.append(String.format("\nThread Pool Size: %d", src.getThreadPoolSize()));
+ sb.append(String.format("\nUnhealthy On Restarting: %b", src.isUnhealthyOnRestarting()));
sb.append(String.format("\nUnhealthy On Exhaust: %b", src.isUnhealthyOnExhausted()));
sb.append("\n\nRoutes:\n");
@@ -187,6 +188,7 @@ public class RouteControllerConsole extends AbstractDevConsole {
root.put("backoffMaxElapsedTime", src.getBackOffMaxElapsedTime());
root.put("backoffMaxAttempts", src.getBackOffMaxAttempts());
root.put("threadPoolSize", src.getThreadPoolSize());
+ root.put("unhealthyOnRestarting", src.isUnhealthyOnRestarting());
root.put("unhealthyOnExhausted", src.isUnhealthyOnExhausted());
root.put("routes", list);
diff --git a/core/camel-health/src/generated/resources/META-INF/services/org/apache/camel/health-check/route-controller-check b/core/camel-health/src/generated/resources/META-INF/services/org/apache/camel/health-check/route-controller-check
new file mode 100644
index 00000000000..d6acdd88860
--- /dev/null
+++ b/core/camel-health/src/generated/resources/META-INF/services/org/apache/camel/health-check/route-controller-check
@@ -0,0 +1,2 @@
+# Generated by camel build tools - do NOT edit this file!
+class=org.apache.camel.impl.health.RouteControllerHealthCheck
diff --git a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
new file mode 100644
index 00000000000..35d37c3bbe1
--- /dev/null
+++ b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.impl.health;
+
+import java.util.Map;
+
+import org.apache.camel.Ordered;
+import org.apache.camel.health.HealthCheckResultBuilder;
+import org.apache.camel.spi.RouteController;
+import org.apache.camel.support.service.ServiceHelper;
+
+/**
+ * Readiness {@link org.apache.camel.health.HealthCheck} for route controller.
+ */
+@org.apache.camel.spi.annotations.HealthCheck("route-controller-check")
+public class RouteControllerHealthCheck extends AbstractHealthCheck {
+
+ public RouteControllerHealthCheck() {
+ super("camel", "route-controller");
+ }
+
+ @Override
+ public int getOrder() {
+ // controller should be early
+ return Ordered.HIGHEST + 1000;
+ }
+
+ @Override
+ protected void doCall(HealthCheckResultBuilder builder, Map<String, Object> options) {
+ boolean up = false;
+
+ RouteController rc = getCamelContext().getRouteController();
+ if (rc != null) {
+ // should be down if starting routes as all routes must be complete before ready
+ up = ServiceHelper.isStarted(rc);
+ if (up) {
+ // is starting routes in progress then we should be DOWN
+ up = !rc.isStartingRoutes();
+ }
+ }
+
+ if (up) {
+ builder.up();
+ } else {
+ builder.detail("route.controller", "Starting routes in progress");
+ builder.down();
+ }
+ }
+
+}
diff --git a/core/camel-main/src/generated/java/org/apache/camel/main/MainConfigurationPropertiesConfigurer.java b/core/camel-main/src/generated/java/org/apache/camel/main/MainConfigurationPropertiesConfigurer.java
index fab74363ba5..35c2a6e7c7a 100644
--- a/core/camel-main/src/generated/java/org/apache/camel/main/MainConfigurationPropertiesConfigurer.java
+++ b/core/camel-main/src/generated/java/org/apache/camel/main/MainConfigurationPropertiesConfigurer.java
@@ -173,6 +173,8 @@ public class MainConfigurationPropertiesConfigurer extends org.apache.camel.supp
case "RouteControllerThreadPoolSize": target.setRouteControllerThreadPoolSize(property(camelContext, int.class, value)); return true;
case "routecontrollerunhealthyonexhausted":
case "RouteControllerUnhealthyOnExhausted": target.setRouteControllerUnhealthyOnExhausted(property(camelContext, boolean.class, value)); return true;
+ case "routecontrollerunhealthyonrestarting":
+ case "RouteControllerUnhealthyOnRestarting": target.setRouteControllerUnhealthyOnRestarting(property(camelContext, boolean.class, value)); return true;
case "routefilterexcludepattern":
case "RouteFilterExcludePattern": target.setRouteFilterExcludePattern(property(camelContext, java.lang.String.class, value)); return true;
case "routefilterincludepattern":
@@ -434,6 +436,8 @@ public class MainConfigurationPropertiesConfigurer extends org.apache.camel.supp
case "RouteControllerThreadPoolSize": return int.class;
case "routecontrollerunhealthyonexhausted":
case "RouteControllerUnhealthyOnExhausted": return boolean.class;
+ case "routecontrollerunhealthyonrestarting":
+ case "RouteControllerUnhealthyOnRestarting": return boolean.class;
case "routefilterexcludepattern":
case "RouteFilterExcludePattern": return java.lang.String.class;
case "routefilterincludepattern":
@@ -696,6 +700,8 @@ public class MainConfigurationPropertiesConfigurer extends org.apache.camel.supp
case "RouteControllerThreadPoolSize": return target.getRouteControllerThreadPoolSize();
case "routecontrollerunhealthyonexhausted":
case "RouteControllerUnhealthyOnExhausted": return target.isRouteControllerUnhealthyOnExhausted();
+ case "routecontrollerunhealthyonrestarting":
+ case "RouteControllerUnhealthyOnRestarting": return target.isRouteControllerUnhealthyOnRestarting();
case "routefilterexcludepattern":
case "RouteFilterExcludePattern": return target.getRouteFilterExcludePattern();
case "routefilterincludepattern":
diff --git a/core/camel-main/src/generated/resources/META-INF/camel-main-configuration-metadata.json b/core/camel-main/src/generated/resources/META-INF/camel-main-configuration-metadata.json
index 5577ce62cb1..1bf93cf0c12 100644
--- a/core/camel-main/src/generated/resources/META-INF/camel-main-configuration-metadata.json
+++ b/core/camel-main/src/generated/resources/META-INF/camel-main-configuration-metadata.json
@@ -93,6 +93,7 @@
{ "name": "camel.main.routeControllerSuperviseEnabled", "description": "To enable using supervising route controller which allows Camel to startup and then the controller takes care of starting the routes in a safe manner. This can be used when you want to startup Camel despite a route may otherwise fail fast during startup and cause Camel to fail to startup as well. By delegating the route startup to the supervising route controller then its manages the startup using a background th [...]
{ "name": "camel.main.routeControllerThreadPoolSize", "description": "The number of threads used by the route controller scheduled thread pool that are used for restarting routes. The pool uses 1 thread by default, but you can increase this to allow the controller to concurrently attempt to restart multiple routes in case more than one route has problems starting.", "sourceType": "org.apache.camel.main.DefaultConfigurationProperties", "type": "integer", "javaType": "int" },
{ "name": "camel.main.routeControllerUnhealthyOnExhausted", "description": "Whether to mark the route as unhealthy (down) when all restarting attempts (backoff) have failed and the route is not successfully started and the route manager is giving up. Setting this to true allows health checks to know about this and can report the Camel application as DOWN. The default is false.", "sourceType": "org.apache.camel.main.DefaultConfigurationProperties", "type": "boolean", "javaType": "bool [...]
+ { "name": "camel.main.routeControllerUnhealthyOnRestarting", "description": "Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled for restarting (backoff). Setting this to true allows health checks to know about this and can report the Camel application as DOWN. The default is false.", "sourceType": "org.apache.camel.main.DefaultConfigurationProperties", "type": "boolean", "javaType": "boolean", "defaultValue": "false" },
{ "name": "camel.main.routeFilterExcludePattern", "description": "Used for filtering routes routes matching the given pattern, which follows the following rules: - Match by route id - Match by route input endpoint uri The matching is using exact match, by wildcard and regular expression as documented by PatternHelper#matchPattern(String,String) . For example to only include routes which starts with foo in their route id's, use: include=foo* And to exclude routes which starts from [...]
{ "name": "camel.main.routeFilterIncludePattern", "description": "Used for filtering routes matching the given pattern, which follows the following rules: - Match by route id - Match by route input endpoint uri The matching is using exact match, by wildcard and regular expression as documented by PatternHelper#matchPattern(String,String) . For example to only include routes which starts with foo in their route id's, use: include=foo* And to exclude routes which starts from JMS en [...]
{ "name": "camel.main.routesBuilderClasses", "description": "Sets classes names that implement RoutesBuilder .", "sourceType": "org.apache.camel.main.MainConfigurationProperties", "type": "string", "javaType": "java.lang.String" },
diff --git a/core/camel-main/src/main/docs/main.adoc b/core/camel-main/src/main/docs/main.adoc
index b4bbd92d60a..be566a90ec8 100644
--- a/core/camel-main/src/main/docs/main.adoc
+++ b/core/camel-main/src/main/docs/main.adoc
@@ -19,7 +19,7 @@ The following tables lists all the options:
// main options: START
=== Camel Main configurations
-The camel.main supports 127 options, which are listed below.
+The camel.main supports 128 options, which are listed below.
[width="100%",cols="2,5,^1,2",options="header"]
|===
@@ -100,6 +100,7 @@ The camel.main supports 127 options, which are listed below.
| *camel.main.routeController{zwsp}SuperviseEnabled* | To enable using supervising route controller which allows Camel to startup and then the controller takes care of starting the routes in a safe manner. This can be used when you want to startup Camel despite a route may otherwise fail fast during startup and cause Camel to fail to startup as well. By delegating the route startup to the supervising route controller then its manages the startup using a background thread. The controller [...]
| *camel.main.routeController{zwsp}ThreadPoolSize* | The number of threads used by the route controller scheduled thread pool that are used for restarting routes. The pool uses 1 thread by default, but you can increase this to allow the controller to concurrently attempt to restart multiple routes in case more than one route has problems starting. | | int
| *camel.main.routeController{zwsp}UnhealthyOnExhausted* | Whether to mark the route as unhealthy (down) when all restarting attempts (backoff) have failed and the route is not successfully started and the route manager is giving up. Setting this to true allows health checks to know about this and can report the Camel application as DOWN. The default is false. | false | boolean
+| *camel.main.routeController{zwsp}UnhealthyOnRestarting* | Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled for restarting (backoff). Setting this to true allows health checks to know about this and can report the Camel application as DOWN. The default is false. | false | boolean
| *camel.main.routeFilterExclude{zwsp}Pattern* | Used for filtering routes routes matching the given pattern, which follows the following rules: - Match by route id - Match by route input endpoint uri The matching is using exact match, by wildcard and regular expression as documented by PatternHelper#matchPattern(String,String) . For example to only include routes which starts with foo in their route id's, use: include=foo* And to exclude routes which starts from JMS endpoints, use: [...]
| *camel.main.routeFilterInclude{zwsp}Pattern* | Used for filtering routes matching the given pattern, which follows the following rules: - Match by route id - Match by route input endpoint uri The matching is using exact match, by wildcard and regular expression as documented by PatternHelper#matchPattern(String,String) . For example to only include routes which starts with foo in their route id's, use: include=foo* And to exclude routes which starts from JMS endpoints, use: exclude [...]
| *camel.main.routesBuilder{zwsp}Classes* | Sets classes names that implement RoutesBuilder . | | String
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationConfigurer.java b/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationConfigurer.java
index a8c0a980f04..09bd2dceab3 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationConfigurer.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationConfigurer.java
@@ -378,6 +378,7 @@ public final class DefaultConfigurationConfigurer {
src.setBackOffMultiplier(config.getRouteControllerBackOffMultiplier());
}
src.setUnhealthyOnExhausted(config.isRouteControllerUnhealthyOnExhausted());
+ src.setUnhealthyOnRestarting(config.isRouteControllerUnhealthyOnRestarting());
}
}
diff --git a/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationProperties.java b/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationProperties.java
index d65aaecc9d1..d918b39705b 100644
--- a/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationProperties.java
+++ b/core/camel-main/src/main/java/org/apache/camel/main/DefaultConfigurationProperties.java
@@ -152,6 +152,7 @@ public abstract class DefaultConfigurationProperties<T> {
private long routeControllerBackOffMaxAttempts;
private double routeControllerBackOffMultiplier;
private boolean routeControllerUnhealthyOnExhausted;
+ private boolean routeControllerUnhealthyOnRestarting;
// startup recorder
@Metadata(enums = "false,off,java-flight-recorder,jfr,logging,backlog")
private String startupRecorder;
@@ -1654,6 +1655,22 @@ public abstract class DefaultConfigurationProperties<T> {
this.routeControllerUnhealthyOnExhausted = routeControllerUnhealthyOnExhausted;
}
+ public boolean isRouteControllerUnhealthyOnRestarting() {
+ return routeControllerUnhealthyOnRestarting;
+ }
+
+ /**
+ * Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled
+ * for restarting (backoff).
+ *
+ * Setting this to true allows health checks to know about this and can report the Camel application as DOWN.
+ *
+ * The default is false.
+ */
+ public void setRouteControllerUnhealthyOnRestarting(boolean routeControllerUnhealthyOnRestarting) {
+ this.routeControllerUnhealthyOnRestarting = routeControllerUnhealthyOnRestarting;
+ }
+
public String getStartupRecorder() {
return startupRecorder;
}
@@ -2878,6 +2895,19 @@ public abstract class DefaultConfigurationProperties<T> {
return (T) this;
}
+ /**
+ * Whether to mark the route as unhealthy (down) when the route failed to initially start, and is being controlled
+ * for restarting (backoff).
+ *
+ * Setting this to true allows health checks to know about this and can report the Camel application as DOWN.
+ *
+ * The default is false.
+ */
+ public T withRouteControllerUnhealthyOnRestarting(boolean routeControllerUnhealthyOnRestarting) {
+ this.routeControllerUnhealthyOnRestarting = routeControllerUnhealthyOnRestarting;
+ return (T) this;
+ }
+
/**
* To use startup recorder for capturing execution time during starting Camel. The recorder can be one of: false (or
* off), logging, backlog, java-flight-recorder (or jfr).
diff --git a/core/camel-util/src/main/java/org/apache/camel/util/backoff/BackOff.java b/core/camel-util/src/main/java/org/apache/camel/util/backoff/BackOff.java
index 7037a2f15d1..3939d50eed8 100644
--- a/core/camel-util/src/main/java/org/apache/camel/util/backoff/BackOff.java
+++ b/core/camel-util/src/main/java/org/apache/camel/util/backoff/BackOff.java
@@ -120,8 +120,12 @@ public final class BackOff {
if (maxElapsedTime != MAX_DURATION) {
sb.append(", maxElapsedTime=").append(maxElapsedTime.toMillis());
}
- sb.append(", maxAttempts=").append(maxAttempts);
- sb.append(", multiplier=").append(multiplier);
+ if (maxAttempts != Long.MAX_VALUE) {
+ sb.append(", maxAttempts=").append(maxAttempts);
+ }
+ if (multiplier != DEFAULT_MULTIPLIER) {
+ sb.append(", multiplier=").append(multiplier);
+ }
sb.append("]");
return sb.toString();
}
diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java
index 08554025b69..e9f501b6576 100644
--- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java
+++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java
@@ -133,7 +133,7 @@ public class RouteControllerAction extends ActionWatchCommand {
row.attempts = jt.getLong("attempts");
long time = jt.getLong("lastAttempt");
if (time > 0) {
- row.lastAttempt = TimeUtils.printDuration(time);
+ row.lastAttempt = TimeUtils.printSince(time);
}
time = jt.getLong("nextAttempt");
if (time > 0) {
@@ -175,7 +175,8 @@ public class RouteControllerAction extends ActionWatchCommand {
System.out.printf("\tBackoff Max Elapsed Time: %d%n", jo.getInteger("backoffMaxElapsedTime"));
System.out.printf("\tBackoff Max Attempts: %d%n", jo.getInteger("backoffMaxAttempts"));
System.out.printf("\tThread Pool Size: %d%n", jo.getInteger("threadPoolSize"));
- System.out.printf("\tUnhealthy on Exhaust: %b%n", jo.getBoolean("unhealthyOnExhausted"));
+ System.out.printf("\tUnhealthy On Restarting: %b%n", jo.getBoolean("unhealthyOnRestarting"));
+ System.out.printf("\tUnhealthy On Exhaust: %b%n", jo.getBoolean("unhealthyOnExhausted"));
System.out.println("\n");
}
dumpTable(rows, true);
@@ -204,6 +205,7 @@ public class RouteControllerAction extends ActionWatchCommand {
new Column().header("STATE").headerAlign(HorizontalAlign.RIGHT).with(this::getSupervising),
new Column().visible(supervised).header("ATTEMPT").headerAlign(HorizontalAlign.CENTER)
.dataAlign(HorizontalAlign.CENTER).with(this::getAttempts),
+ new Column().visible(supervised).header("ELAPSED").headerAlign(HorizontalAlign.CENTER).with(this::getElapsed),
new Column().visible(supervised).header("LAST-AGO").headerAlign(HorizontalAlign.CENTER).with(this::getLast),
new Column().visible(supervised).header("ERROR-MESSAGE").headerAlign(HorizontalAlign.LEFT)
.dataAlign(HorizontalAlign.LEFT)
@@ -291,11 +293,14 @@ public class RouteControllerAction extends ActionWatchCommand {
protected String getLast(Row r) {
if (r.lastAttempt != null && !r.lastAttempt.isEmpty()) {
- String s = r.lastAttempt;
- if (r.elapsed != null && !r.elapsed.isEmpty()) {
- s += " (" + r.elapsed + ")";
- }
- return s;
+ return r.lastAttempt;
+ }
+ return "";
+ }
+
+ protected String getElapsed(Row r) {
+ if (r.elapsed != null && !r.elapsed.isEmpty()) {
+ return r.elapsed;
}
return "";
}
(camel) 03/07: CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch rc
in repository https://gitbox.apache.org/repos/asf/camel.git
commit 070e68f937e828babab2779e624a8f1e4607b3f2
Author: Claus Ibsen <cl...@gmail.com>
AuthorDate: Sat Dec 16 13:08:09 2023 +0100
CAMEL-20242: camel-core: RouteController health check to be DOWN during starting routes. Supervising route controller option to be DOWN during restarting phase.
---
.../src/main/java/org/apache/camel/spi/RouteController.java | 6 ++++++
.../java/org/apache/camel/spi/SupervisingRouteController.java | 4 ++++
.../java/org/apache/camel/impl/engine/DefaultRouteController.java | 5 +++++
.../camel/impl/engine/DefaultSupervisingRouteController.java | 7 ++++++-
.../org/apache/camel/impl/engine/InternalRouteController.java | 7 +++++++
.../org/apache/camel/impl/console/RouteControllerConsole.java | 4 ++++
.../org/apache/camel/impl/health/RouteControllerHealthCheck.java | 8 ++------
.../dsl/jbang/core/commands/action/RouteControllerAction.java | 3 +++
8 files changed, 37 insertions(+), 7 deletions(-)
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java b/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java
index 5b7eefcff34..d538eb77d97 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/RouteController.java
@@ -99,6 +99,12 @@ public interface RouteController extends CamelContextAware, StaticService {
*/
boolean isStartingRoutes();
+ /**
+ * Indicates if the route controller has routes that are unhealthy such as they have not yet been successfully started,
+ * and if being supervised then the route can either be pending restarts or failed all restart attempts and are exhausted.
+ */
+ boolean isUnhealthyRoutes();
+
/**
* Reloads all the routes
*
diff --git a/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java b/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
index 4631919cfbc..a2018986e1b 100644
--- a/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
+++ b/core/camel-api/src/main/java/org/apache/camel/spi/SupervisingRouteController.java
@@ -173,6 +173,10 @@ public interface SupervisingRouteController extends RouteController {
*/
Throwable getRestartException(String routeId);
+ /**
+ * Whether the route controller is currently starting routes for the first time.
+ * This only reports on the first time start phase.
+ */
boolean isStartingRoutes();
}
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRouteController.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRouteController.java
index 54ee46708f4..fa63220eedb 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRouteController.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultRouteController.java
@@ -108,6 +108,11 @@ public class DefaultRouteController extends ServiceSupport implements RouteContr
return getInternalRouteController().isStartingRoutes();
}
+ @Override
+ public boolean isUnhealthyRoutes() {
+ return getInternalRouteController().isUnhealthyRoutes();
+ }
+
@Override
public void reloadAllRoutes() throws Exception {
getInternalRouteController().reloadAllRoutes();
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
index 44b3b6cc6a0..5df8119b4fc 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/DefaultSupervisingRouteController.java
@@ -248,7 +248,7 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
// *********************************
@Override
- public boolean isStartingRoutes() {
+ public boolean isUnhealthyRoutes() {
boolean answer = startingRoutes;
// if we have started the routes first time, but some failed and are scheduled for restart
@@ -264,6 +264,11 @@ public class DefaultSupervisingRouteController extends DefaultRouteController im
return answer;
}
+ @Override
+ public boolean isStartingRoutes() {
+ return startingRoutes;
+ }
+
@Override
public void startRoute(String routeId) throws Exception {
final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst();
diff --git a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java
index 4341050bbe8..e07a99760cd 100644
--- a/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java
+++ b/core/camel-base-engine/src/main/java/org/apache/camel/impl/engine/InternalRouteController.java
@@ -27,6 +27,8 @@ import org.apache.camel.spi.RouteController;
import org.apache.camel.spi.RouteError;
import org.apache.camel.spi.SupervisingRouteController;
+import static org.apache.camel.support.service.ServiceHelper.isStarted;
+
/**
* Internal {@link RouteController} used internally by {@link AbstractCamelContext}.
*/
@@ -111,6 +113,11 @@ class InternalRouteController implements RouteController {
return abstractCamelContext.isStartingRoutes();
}
+ @Override
+ public boolean isUnhealthyRoutes() {
+ return false;
+ }
+
@Override
public ServiceStatus getRouteStatus(String routeId) {
return abstractCamelContext.getRouteStatus(routeId);
diff --git a/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java b/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java
index c43e1fa1960..bb64f241393 100644
--- a/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java
+++ b/core/camel-console/src/main/java/org/apache/camel/impl/console/RouteControllerConsole.java
@@ -66,6 +66,8 @@ public class RouteControllerConsole extends AbstractDevConsole {
long started = routes.stream().filter(r -> src.getRouteStatus(r.getRouteId()).isStarted())
.count();
+ sb.append(String.format("\nInitial Starting Routes: %b", src.isStartingRoutes()));
+ sb.append(String.format("\nUnhealthy Routes: %b", src.isUnhealthyRoutes()));
sb.append(String.format("Total Routes: %d", routes.size()));
sb.append(String.format("\nStarted Routes: %d", started));
sb.append(String.format("\nRestarting Routes: %d", src.getRestartingRoutes().size()));
@@ -178,6 +180,8 @@ public class RouteControllerConsole extends AbstractDevConsole {
.count();
root.put("controller", "SupervisingRouteController");
+ root.put("startingRoutes", src.isStartingRoutes());
+ root.put("unhealthyRoutes", src.isUnhealthyRoutes());
root.put("totalRoutes", routes.size());
root.put("startedRoutes", started);
root.put("restartingRoutes", src.getRestartingRoutes().size());
diff --git a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
index 35d37c3bbe1..06b076e44f1 100644
--- a/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
+++ b/core/camel-health/src/main/java/org/apache/camel/impl/health/RouteControllerHealthCheck.java
@@ -45,12 +45,8 @@ public class RouteControllerHealthCheck extends AbstractHealthCheck {
RouteController rc = getCamelContext().getRouteController();
if (rc != null) {
- // should be down if starting routes as all routes must be complete before ready
- up = ServiceHelper.isStarted(rc);
- if (up) {
- // is starting routes in progress then we should be DOWN
- up = !rc.isStartingRoutes();
- }
+ // should only be up if there are no unhealthy routes
+ up = !rc.isUnhealthyRoutes();
}
if (up) {
diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java
index e9f501b6576..8a85c25459a 100644
--- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java
+++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/RouteControllerAction.java
@@ -165,6 +165,8 @@ public class RouteControllerAction extends ActionWatchCommand {
if (supervising) {
if (header) {
System.out.println("Supervising Route Controller");
+ System.out.printf("\tInitial Starting Routes: %b%n", jo.getBoolean("startingRoutes"));
+ System.out.printf("\tUnhealthy Routes: %b%n", jo.getBoolean("unhealthyRoutes"));
System.out.printf("\tRoutes Total: %s%n", jo.getInteger("totalRoutes"));
System.out.printf("\tRoutes Started: %d%n", jo.getInteger("startedRoutes"));
System.out.printf("\tRoutes Restarting: %d%n", jo.getInteger("restartingRoutes"));
@@ -183,6 +185,7 @@ public class RouteControllerAction extends ActionWatchCommand {
} else {
if (header) {
System.out.println("Default Route Controller");
+ System.out.printf("\tStarting Routes: %b%n", jo.getBoolean("startingRoutes"));
System.out.printf("\tRoutes Total: %s%n", jo.getInteger("totalRoutes"));
System.out.println("\n");
}