You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@felix.apache.org by gh...@apache.org on 2020/03/28 00:53:09 UTC
[felix-dev] 03/03: FELIX-6250 Introduce HealthCheckMonitor for
monitoring health checks
This is an automated email from the ASF dual-hosted git repository.
ghenzler pushed a commit to branch feature/FELIX-6245-condition-service-interfaces-FELIX-6250-healthcheck-monitor
in repository https://gitbox.apache.org/repos/asf/felix-dev.git
commit 775545aa8f0657d7c5f703bc901693dcdbaff92f
Author: georg.henzler <ge...@netcentric.biz>
AuthorDate: Sat Mar 28 01:46:50 2020 +0100
FELIX-6250 Introduce HealthCheckMonitor for monitoring health checks
---
healthcheck/core/pom.xml | 2 +-
.../impl/executor/CombinedExecutionResult.java | 9 +
.../hc/core/impl/monitor/HealthCheckMonitor.java | 344 +++++++++++++++++++++
3 files changed, 354 insertions(+), 1 deletion(-)
diff --git a/healthcheck/core/pom.xml b/healthcheck/core/pom.xml
index 36333a5..620d11f 100644
--- a/healthcheck/core/pom.xml
+++ b/healthcheck/core/pom.xml
@@ -145,7 +145,7 @@
<dependency>
<groupId>org.apache.felix</groupId>
<artifactId>org.apache.felix.healthcheck.api</artifactId>
- <version>2.0.0</version>
+ <version>2.0.3-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
diff --git a/healthcheck/core/src/main/java/org/apache/felix/hc/core/impl/executor/CombinedExecutionResult.java b/healthcheck/core/src/main/java/org/apache/felix/hc/core/impl/executor/CombinedExecutionResult.java
index a159fc6..55a9172 100644
--- a/healthcheck/core/src/main/java/org/apache/felix/hc/core/impl/executor/CombinedExecutionResult.java
+++ b/healthcheck/core/src/main/java/org/apache/felix/hc/core/impl/executor/CombinedExecutionResult.java
@@ -89,5 +89,14 @@ public class CombinedExecutionResult implements HealthCheckExecutionResult {
public HealthCheckMetadata getHealthCheckMetadata() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public String toString() {
+ return "CombinedExecutionResult [size="+executionResults.size()+" overall status=" + getHealthCheckResult().getStatus() +
+ ", finishedAt=" + getFinishedAt() +
+ ", elapsedTimeInMs=" + getElapsedTimeInMs() +
+ ", timedOut=" + hasTimedOut() +
+ "]";
+ }
}
\ No newline at end of file
diff --git a/healthcheck/core/src/main/java/org/apache/felix/hc/core/impl/monitor/HealthCheckMonitor.java b/healthcheck/core/src/main/java/org/apache/felix/hc/core/impl/monitor/HealthCheckMonitor.java
new file mode 100644
index 0000000..18b47cb
--- /dev/null
+++ b/healthcheck/core/src/main/java/org/apache/felix/hc/core/impl/monitor/HealthCheckMonitor.java
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The SF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.felix.hc.core.impl.monitor;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Dictionary;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.felix.hc.api.Result;
+import org.apache.felix.hc.api.condition.Healthy;
+import org.apache.felix.hc.api.condition.SystemReady;
+import org.apache.felix.hc.api.condition.Unhealthy;
+import org.apache.felix.hc.api.execution.HealthCheckExecutionResult;
+import org.apache.felix.hc.api.execution.HealthCheckExecutor;
+import org.apache.felix.hc.api.execution.HealthCheckSelector;
+import org.apache.felix.hc.core.impl.executor.CombinedExecutionResult;
+import org.apache.felix.hc.core.impl.executor.HealthCheckExecutorThreadPool;
+import org.apache.felix.hc.core.impl.scheduling.AsyncIntervalJob;
+import org.apache.felix.hc.core.impl.scheduling.AsyncJob;
+import org.apache.felix.hc.core.impl.scheduling.AsyncQuartzCronJob;
+import org.apache.felix.hc.core.impl.scheduling.QuartzCronSchedulerProvider;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.InvalidSyntaxException;
+import org.osgi.framework.ServiceRegistration;
+import org.osgi.service.component.ComponentConstants;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.ConfigurationPolicy;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.event.Event;
+import org.osgi.service.event.EventAdmin;
+import org.osgi.service.metatype.annotations.AttributeDefinition;
+import org.osgi.service.metatype.annotations.Designate;
+import org.osgi.service.metatype.annotations.ObjectClassDefinition;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Monitors health check tags and/or names and depending on configuration:
+ * <p>
+ * <ul>
+ * <li>Activates the condition marker services {@link SystemReady}, {@link Healthy}, {@link Unhealthy}</li>
+ * <li>Sends OSGi events</li>
+ * </ul>
+ * <p>
+ *
+ */
+@Component(immediate = true, configurationPolicy = ConfigurationPolicy.REQUIRE)
+@Designate(ocd = HealthCheckMonitor.Config.class, factory = true)
+public class HealthCheckMonitor implements Runnable {
+ private static final Logger LOG = LoggerFactory.getLogger(HealthCheckMonitor.class);
+
+ public static final String TAG_SYSTEMREADY = "systemready";
+
+ public static final String EVENT_TOPIC_PREFIX = "org/apache/felix/healthchange";
+ public static final String EVENT_PROP_EXECUTION_RESULT = "executionResult";
+ public static final String EVENT_PROP_STATUS = "status";
+ public static final String EVENT_PROP_PREVIOUS_STATUS = "previousStatus";
+
+ private static final Healthy MARKER_SERVICE_HEALTHY = new Healthy() { };
+ private static final Unhealthy MARKER_SERVICE_UNHEALTHY = new Unhealthy() { };
+ private static final SystemReady MARKER_SERVICE_SYSTEMREADY = new SystemReady() { };
+
+
+ @ObjectClassDefinition(name = "Health Check Monitor", description = "Regularly executes health checks according to given interval/cron expression")
+ public @interface Config {
+
+ @AttributeDefinition(name = "Tags", description = "List of tags to query regularly")
+ String[] tags() default {};
+
+ @AttributeDefinition(name = "Names", description = "List of health check names to query regularly")
+ String[] names() default {};
+
+ @AttributeDefinition(name = "Interval (Sec)", description = "Will execute the checks for give tags every n seconds (either use intervalInSec or cronExpression )")
+ long intervalInSec() default 0;
+
+ @AttributeDefinition(name = "Interval (Cron Expresson)", description = "Will execute the checks for give tags according to cron expression")
+ String cronExpression() default "";
+
+ @AttributeDefinition(name = "Register Healthy Marker Service", description = "For the case a given tag/name is healthy, will register a service Healthy with property tag=<tagname> (or name=<hc.name>) that other services can depend on")
+ boolean registerHealthyMarkerService() default true;
+
+ @AttributeDefinition(name = "Register Unhealthy Marker Service", description = "For the case a given tag/name is unhealthy, will register a service Unhealthy with property tag=<tagname> (or name=<hc.name>) that other services can depend on")
+ boolean registerUnhealthyMarkerService() default false;
+
+ @AttributeDefinition(name = "Treat WARN as Healthy", description = "Whether to treat status WARN as healthy (it normally should because WARN indicates a working system that only possibly might become unavailable if no action is taken")
+ boolean treatWarnAsHealthy() default true;
+
+ @AttributeDefinition(name = "Send Events", description = "Whether to send OSGi events for the case a status has changed")
+ boolean sendEvents() default true;
+
+ @AttributeDefinition
+ String webconsole_configurationFactory_nameHint() default "Health Monitor for '{tags}'/'{names}', {intervalInSec}sec/{cronExpression}, Marker Service Healthy:{registerHealthyMarkerService} Unhealthy:{registerUnhealthyMarkerService}, Send Events {sendEvents}";
+ }
+
+ private List<String> tags;
+ private List<String> names;
+ private List<HealthState> healthStates = new ArrayList<>();
+
+ private long intervalInSec;
+ private String cronExpression;
+
+ private boolean registerHealthyMarkerService;
+ private boolean registerUnhealthyMarkerService;
+
+ private boolean treatWarnAsHealthy;
+
+ private boolean sendEvents;
+
+ @Reference
+ HealthCheckExecutorThreadPool healthCheckExecutorThreadPool;
+
+ @Reference
+ QuartzCronSchedulerProvider quartzCronSchedulerProvider;
+
+ AsyncJob monitorJob = null;
+
+ @Reference
+ private HealthCheckExecutor executor;
+
+ @Reference
+ private EventAdmin eventAdmin;
+
+ private BundleContext bundleContext;
+
+ @Activate
+ protected final void activate(BundleContext bundleContext, Config config, ComponentContext componentContext) throws InvalidSyntaxException {
+
+ this.bundleContext = bundleContext;
+
+ this.tags = Arrays.asList(config.tags());
+ this.tags.stream().filter(StringUtils::isNotBlank).forEach(tag -> healthStates.add(new HealthState(tag, true)));
+
+ this.names = Arrays.asList(config.names());
+ this.names.stream().filter(StringUtils::isNotBlank).forEach(name -> healthStates.add(new HealthState(name, false)));
+
+ this.registerHealthyMarkerService = config.registerHealthyMarkerService();
+ this.registerUnhealthyMarkerService = config.registerUnhealthyMarkerService();
+
+ this.treatWarnAsHealthy = config.treatWarnAsHealthy();
+ this.sendEvents = config.sendEvents();
+
+ this.intervalInSec = config.intervalInSec();
+ this.cronExpression = config.cronExpression();
+ if (StringUtils.isNotBlank(cronExpression)) {
+ try {
+ monitorJob = new AsyncQuartzCronJob(this, quartzCronSchedulerProvider, "job-hc-monitor-"+componentContext.getProperties().get(ComponentConstants.COMPONENT_ID), "healthcheck-monitor", cronExpression);
+ } catch(ClassNotFoundException e) {
+ throw new IllegalArgumentException("Cannot use cron expression "+cronExpression+" while class is not available: "+cronExpression);
+ }
+ } else if (intervalInSec > 0) {
+ monitorJob = new AsyncIntervalJob(this, healthCheckExecutorThreadPool, intervalInSec);
+ } else {
+ throw new IllegalArgumentException("Either cronExpression or intervalInSec needs to be set");
+ }
+ monitorJob.schedule();
+ LOG.info("HealthCheckMonitor active for tags {} and names {}", this.tags, this.names);
+ }
+
+ @Override
+ public String toString() {
+ return "[HealthCheckMonitor tags=" + tags + "/names=" + names + ", intervalInSec="+intervalInSec+"/cron="+cronExpression+"]";
+ }
+
+ @Deactivate
+ protected final void deactivate() {
+ healthStates.stream().forEach(HealthState::cleanUp);
+ monitorJob.unschedule();
+ LOG.info("HealthCheckMonitor deactivated for tags {} and names {}", this.tags, this.names);
+ }
+
+ public void run() {
+ try {
+ // run in tags/names in parallel
+ healthStates.parallelStream().forEach((healthState) -> {
+ HealthCheckSelector selector = healthState.isTag ? HealthCheckSelector.tags(healthState.tagOrName)
+ : HealthCheckSelector.names(healthState.tagOrName);
+
+ List<HealthCheckExecutionResult> executionResults = executor.execute(selector);
+
+ HealthCheckExecutionResult result = executionResults.size() == 1 ? executionResults.get(0)
+ : new CombinedExecutionResult(executionResults, Result.Status.TEMPORARILY_UNAVAILABLE);
+
+ LOG.trace("Result of '{}' => {}", healthState.tagOrName, result.getHealthCheckResult().getStatus());
+
+ healthState.update(result);
+ });
+
+ LOG.trace("HealthCheckMonitor: updated results for tags {} and names {}", this.tags, this.names);
+ } catch(Exception e) {
+ LOG.error("Exception HealthCheckMonitor run(): "+e, e);
+ }
+
+ }
+
+ private class HealthState {
+
+ private String tagOrName;
+ private boolean isTag;
+ private String propertyName;
+
+ private ServiceRegistration<?> healthyRegistration = null;
+ private ServiceRegistration<Unhealthy> unhealthyRegistration = null;
+
+ private Result.Status status = null;
+ private boolean isHealthy = false;
+ private boolean statusChanged = false;
+
+ public HealthState(String tagOrName, boolean isTag) {
+ this.tagOrName = tagOrName;
+ this.isTag = isTag;
+ this.propertyName = isTag ? "tag" : "name";
+ }
+
+ synchronized void update(HealthCheckExecutionResult executionResult) {
+ Result.Status previousStatus = status;
+ status = executionResult.getHealthCheckResult().getStatus();
+
+ isHealthy = (status == Result.Status.OK || (treatWarnAsHealthy && status == Result.Status.WARN));
+ statusChanged = previousStatus != status;
+ LOG.trace(" {}: isHealthy={} statusChanged={}", tagOrName, isHealthy, statusChanged);
+
+ registerMarkerServices();
+ sendEvent(executionResult, previousStatus);
+ }
+
+ private void registerMarkerServices() {
+ if (registerHealthyMarkerService) {
+ if (isHealthy && healthyRegistration == null) {
+ registerHealthyService();
+ } else if (!isHealthy && healthyRegistration != null) {
+ unregisterHealthyService();
+ }
+ }
+ if (registerUnhealthyMarkerService) {
+ if (!isHealthy && unhealthyRegistration == null) {
+ registerUnhealthyService();
+ } else if (isHealthy && unhealthyRegistration != null) {
+ unregisterUnhealthyService();
+ }
+ }
+ }
+
+ private void registerHealthyService() {
+ if (healthyRegistration == null) {
+ LOG.debug("HealthCheckMonitor: registerHealthyService() {} ", tagOrName);
+ Dictionary<String, String> registrationProps = new Hashtable<>();
+ registrationProps.put(propertyName, tagOrName);
+ registrationProps.put("activated", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));
+
+ if (StringUtils.equals(tagOrName, TAG_SYSTEMREADY)) {
+ LOG.debug("HealthCheckMonitor: SYSTEM READY");
+ healthyRegistration = bundleContext.registerService(
+ new String[] { SystemReady.class.getName(), Healthy.class.getName()},
+ MARKER_SERVICE_SYSTEMREADY, registrationProps);
+ } else {
+ healthyRegistration = bundleContext.registerService(Healthy.class, MARKER_SERVICE_HEALTHY,
+ registrationProps);
+ }
+ LOG.debug("HealthCheckMonitor: Healthy service for {} '{}' registered", propertyName, tagOrName);
+ }
+ }
+
+ private void unregisterHealthyService() {
+ if (healthyRegistration != null) {
+ healthyRegistration.unregister();
+ healthyRegistration = null;
+ LOG.debug("HealthCheckMonitor: Healthy service for {} '{}' unregistered", propertyName, tagOrName);
+ }
+ }
+
+ private void registerUnhealthyService() {
+ if (unhealthyRegistration == null) {
+ Dictionary<String, String> registrationProps = new Hashtable<>();
+ registrationProps.put("tag", tagOrName);
+ registrationProps.put("activated", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));
+ unhealthyRegistration = bundleContext.registerService(Unhealthy.class, MARKER_SERVICE_UNHEALTHY,
+ registrationProps);
+ LOG.debug("HealthCheckMonitor: Unhealthy service for {} '{}' registered", propertyName, tagOrName);
+ }
+ }
+
+ private void unregisterUnhealthyService() {
+ if (unhealthyRegistration != null) {
+ unhealthyRegistration.unregister();
+ unhealthyRegistration = null;
+ LOG.debug("HealthCheckMonitor: Unhealthy service for {} '{}' unregistered", propertyName, tagOrName);
+ }
+ }
+
+
+ private void sendEvent(HealthCheckExecutionResult executionResult, Result.Status previousStatus) {
+ if(sendEvents && statusChanged) {
+ Map<String,Object> properties = new HashMap<>();
+ properties.put(EVENT_PROP_STATUS , status);
+ if(previousStatus != null) {
+ properties.put(EVENT_PROP_PREVIOUS_STATUS , previousStatus);
+ }
+ properties.put(EVENT_PROP_EXECUTION_RESULT, executionResult);
+ String topic = EVENT_TOPIC_PREFIX + "/" +propertyName + "/" + tagOrName.replaceAll("\\s+","_");
+ eventAdmin.postEvent(new Event(topic, properties));
+ LOG.debug("HealthCheckMonitor: Posted event for topic '{}': Status change from {} to {}", topic, previousStatus, status);
+ if(!(executionResult instanceof CombinedExecutionResult)) {
+ String componentClass = (String) executionResult.getHealthCheckMetadata().getServiceReference().getProperty(ComponentConstants.COMPONENT_NAME);
+ if(StringUtils.isNotBlank(componentClass)) {
+ String topicClass = EVENT_TOPIC_PREFIX + "/class/" + componentClass.replace(".", "/");
+ eventAdmin.postEvent(new Event(topicClass, properties));
+ LOG.debug("HealthCheckMonitor: Posted event for topic '{}': Status change from {} to {}", topicClass, previousStatus, status);
+ }
+ }
+ }
+ }
+
+ synchronized void cleanUp() {
+ unregisterHealthyService();
+ unregisterUnhealthyService();
+ }
+
+ }
+}