You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@karaf.apache.org by jb...@apache.org on 2017/07/12 04:40:39 UTC
karaf-decanter git commit: [KARAF-4645] Alerts are now throttable to
avoid spamming on the alerters
Repository: karaf-decanter
Updated Branches:
refs/heads/master 261b10d98 -> 94f5f6a81
[KARAF-4645] Alerts are now throttable to avoid spamming on the alerters
Project: http://git-wip-us.apache.org/repos/asf/karaf-decanter/repo
Commit: http://git-wip-us.apache.org/repos/asf/karaf-decanter/commit/94f5f6a8
Tree: http://git-wip-us.apache.org/repos/asf/karaf-decanter/tree/94f5f6a8
Diff: http://git-wip-us.apache.org/repos/asf/karaf-decanter/diff/94f5f6a8
Branch: refs/heads/master
Commit: 94f5f6a8112d7979b251768ce2ed339dbda446be
Parents: 261b10d
Author: Jean-Baptiste Onofré <jb...@apache.org>
Authored: Wed Jul 12 06:39:36 2017 +0200
Committer: Jean-Baptiste Onofré <jb...@apache.org>
Committed: Wed Jul 12 06:40:29 2017 +0200
----------------------------------------------------------------------
.../karaf/decanter/sla/camel/CamelAlerter.java | 1 +
.../karaf/decanter/sla/checker/AlertStore.java | 27 ++++++++
.../decanter/sla/checker/AlertStoreImpl.java | 67 ++++++++++++++++++++
.../karaf/decanter/sla/checker/Checker.java | 47 +++++++++++---
.../karaf/decanter/sla/email/EmailAlerter.java | 13 +++-
.../apache/karaf/decanter/sla/log/Logger.java | 16 ++++-
6 files changed, 158 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/karaf-decanter/blob/94f5f6a8/sla/camel/src/main/java/org/apache/karaf/decanter/sla/camel/CamelAlerter.java
----------------------------------------------------------------------
diff --git a/sla/camel/src/main/java/org/apache/karaf/decanter/sla/camel/CamelAlerter.java b/sla/camel/src/main/java/org/apache/karaf/decanter/sla/camel/CamelAlerter.java
index 16e4dad..1e60759 100644
--- a/sla/camel/src/main/java/org/apache/karaf/decanter/sla/camel/CamelAlerter.java
+++ b/sla/camel/src/main/java/org/apache/karaf/decanter/sla/camel/CamelAlerter.java
@@ -63,6 +63,7 @@ public class CamelAlerter implements EventHandler {
headers.put("alertLevel", event.getProperty("alertLevel"));
headers.put("alertAttribute", event.getProperty("alertAttribute"));
headers.put("alertPattern", event.getProperty("alertPattern"));
+ headers.put("alertBackToNormal", event.getProperty("alertBackToNormal"));
producerTemplate.sendBodyAndHeaders(alertDestinationUri, data, headers);
}
http://git-wip-us.apache.org/repos/asf/karaf-decanter/blob/94f5f6a8/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStore.java
----------------------------------------------------------------------
diff --git a/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStore.java b/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStore.java
new file mode 100644
index 0000000..0179ada
--- /dev/null
+++ b/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStore.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.karaf.decanter.sla.checker;
+
+public interface AlertStore {
+
+ void add(String name, String level);
+
+ void remove(String name, String level);
+
+ boolean known(String name, String level);
+
+}
http://git-wip-us.apache.org/repos/asf/karaf-decanter/blob/94f5f6a8/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStoreImpl.java
----------------------------------------------------------------------
diff --git a/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStoreImpl.java b/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStoreImpl.java
new file mode 100644
index 0000000..696d198
--- /dev/null
+++ b/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/AlertStoreImpl.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.karaf.decanter.sla.checker;
+
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+
+import java.util.HashSet;
+import java.util.Set;
+
+@Component(
+ immediate = true
+)
+public class AlertStoreImpl implements AlertStore {
+
+ private Set<String> errorAlerts;
+ private Set<String> warnAlerts;
+
+ @Activate
+ public void activate() {
+ this.errorAlerts = new HashSet<>();
+ this.warnAlerts = new HashSet<>();
+ }
+
+ public void add(String name, String level) {
+ if (level.equals("error")) {
+ this.errorAlerts.add(name);
+ }
+ if (level.equals("warn")) {
+ this.warnAlerts.add(name);
+ }
+ }
+
+ public void remove(String name, String level) {
+ if (level.equals("error")) {
+ this.errorAlerts.remove(name);
+ }
+ if (level.equals("warn")) {
+ this.warnAlerts.remove(name);
+ }
+ }
+
+ public boolean known(String name, String level) {
+ if (level.equals("error")) {
+ return this.errorAlerts.contains(name);
+ }
+ if (level.equals("warn")) {
+ return this.warnAlerts.contains(name);
+ }
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/karaf-decanter/blob/94f5f6a8/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/Checker.java
----------------------------------------------------------------------
diff --git a/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/Checker.java b/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/Checker.java
index d254846..8d38430 100644
--- a/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/Checker.java
+++ b/sla/checker/src/main/java/org/apache/karaf/decanter/sla/checker/Checker.java
@@ -28,9 +28,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
-import java.util.Dictionary;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -49,6 +47,8 @@ public class Checker implements EventHandler {
private Dictionary<String, Object> config;
private EventAdmin eventAdmin;
+ private AlertStore alertStore;
+
@SuppressWarnings("unchecked")
@Activate
public void activate(ComponentContext context) {
@@ -70,8 +70,20 @@ public class Checker implements EventHandler {
if (errorPattern != null) {
Object value = collectEvent.getProperty(name);
if (!validate(errorPattern, value)) {
- Event alertEvent = populateAlertEvent("error", collectEvent, name, errorPattern);
- eventAdmin.postEvent(alertEvent);
+ if (!alertStore.known(name, "error")) {
+ alertStore.add(name, "error");
+ Event alertEvent = populateAlertEvent("error", collectEvent, name, errorPattern, false);
+ eventAdmin.postEvent(alertEvent);
+ }
+ } else {
+ if (alertStore.known(name, "error")) {
+ eventAdmin.postEvent(populateAlertEvent("error", collectEvent, name, errorPattern, true));
+ alertStore.remove(name, "error");
+ }
+ }
+ } else {
+ if (alertStore.known(name, "error")) {
+ eventAdmin.postEvent(populateAlertEvent("error", collectEvent, name, "REMOVED", true));
}
}
@@ -85,18 +97,32 @@ public class Checker implements EventHandler {
if (warnPattern != null) {
Object value = collectEvent.getProperty(name);
if (!validate(warnPattern, value)) {
- Event alertEvent = populateAlertEvent("warn", collectEvent, name, warnPattern);
- eventAdmin.postEvent(alertEvent);
+ if (!alertStore.known(name, "warn")) {
+ alertStore.add(name, "warn");
+ Event alertEvent = populateAlertEvent("warn", collectEvent, name, warnPattern, false);
+ eventAdmin.postEvent(alertEvent);
+ }
+ } else {
+ if (alertStore.known(name, "warn")) {
+ eventAdmin.postEvent(populateAlertEvent("warn", collectEvent, name, warnPattern, true));
+ alertStore.remove(name, "warn");
+ }
+ }
+ } else {
+ if (alertStore.known(name, "warn")) {
+ eventAdmin.postEvent(populateAlertEvent("warn", collectEvent, name, "REMOVED", true));
+ alertStore.remove(name, "warn");
}
}
}
}
- private Event populateAlertEvent(String level, Event collectEvent, String attribute, String pattern) {
+ private Event populateAlertEvent(String level, Event collectEvent, String attribute, String pattern, boolean recovery) {
Map<String, Object> data = new HashMap<>();
data.put("alertLevel", level);
data.put("alertAttribute", attribute);
data.put("alertPattern", pattern);
+ data.put("alertBackToNormal", recovery);
for (String name : collectEvent.getPropertyNames()) {
data.put(name, collectEvent.getProperty(name));
}
@@ -450,5 +476,10 @@ public class Checker implements EventHandler {
public void setEventAdmin(EventAdmin eventAdmin) {
this.eventAdmin = eventAdmin;
}
+
+ @Reference
+ public void setAlertStore(AlertStore alertStore) {
+ this.alertStore = alertStore;
+ }
}
http://git-wip-us.apache.org/repos/asf/karaf-decanter/blob/94f5f6a8/sla/email/src/main/java/org/apache/karaf/decanter/sla/email/EmailAlerter.java
----------------------------------------------------------------------
diff --git a/sla/email/src/main/java/org/apache/karaf/decanter/sla/email/EmailAlerter.java b/sla/email/src/main/java/org/apache/karaf/decanter/sla/email/EmailAlerter.java
index cdaf01a..265190e 100644
--- a/sla/email/src/main/java/org/apache/karaf/decanter/sla/email/EmailAlerter.java
+++ b/sla/email/src/main/java/org/apache/karaf/decanter/sla/email/EmailAlerter.java
@@ -89,9 +89,18 @@ public class EmailAlerter implements EventHandler {
String alertLevel = (String) event.getProperty("alertLevel");
String alertAttribute = (String) event.getProperty("alertAttribute");
String alertPattern = (String) event.getProperty("alertPattern");
- message.setSubject("[" + alertLevel + "] Alert on " + alertAttribute);
+ boolean recovery = (boolean) event.getProperty("alertBackToNormal");
+ if (!recovery) {
+ message.setSubject("[" + alertLevel + "] Alert on " + alertAttribute);
+ } else {
+ message.setSubject("Alert on " + alertAttribute + " back to normal");
+ }
StringBuilder builder = new StringBuilder();
- builder.append(alertLevel + " alert: " + alertAttribute + " is out of the pattern " + alertPattern + "\n");
+ if (!recovery) {
+ builder.append(alertLevel + " alert: " + alertAttribute + " is out of the pattern " + alertPattern + "\n");
+ } else {
+ builder.append(alertLevel + " alert: " + alertAttribute + " was out of the pattern " + alertPattern + ", but back to normal now\n");
+ }
builder.append("\n");
builder.append("Details:\n");
for (String name : event.getPropertyNames()) {
http://git-wip-us.apache.org/repos/asf/karaf-decanter/blob/94f5f6a8/sla/log/src/main/java/org/apache/karaf/decanter/sla/log/Logger.java
----------------------------------------------------------------------
diff --git a/sla/log/src/main/java/org/apache/karaf/decanter/sla/log/Logger.java b/sla/log/src/main/java/org/apache/karaf/decanter/sla/log/Logger.java
index cf54fb9..ce3ff65 100644
--- a/sla/log/src/main/java/org/apache/karaf/decanter/sla/log/Logger.java
+++ b/sla/log/src/main/java/org/apache/karaf/decanter/sla/log/Logger.java
@@ -37,11 +37,21 @@ public class Logger implements EventHandler {
for (String innerKey : event.getPropertyNames()) {
builder.append(innerKey).append(":").append(event.getProperty(innerKey).toString()).append(" | ");
}
+ boolean backToNormal = (boolean) event.getProperty("alertBackToNormal");
if (event.getProperty("alertLevel") != null && ((String) event.getProperty("alertLevel")).equalsIgnoreCase("error")) {
- LOGGER.error("DECANTER SLA ALERT: {} out of pattern {}", event.getProperty("alertAttribute"), event.getProperty("alertPattern"));
+ if (backToNormal) {
+ LOGGER.info("DECANTER SLA ALERT BACK TO NORMAL: {} was out of pattern {}", event.getProperty("alertAttribute"), event.getProperty("alertPattern"));
+ } else {
+ LOGGER.error("DECANTER SLA ALERT: {} out of pattern {}", event.getProperty("alertAttribute"), event.getProperty("alertPattern"));
+ }
LOGGER.error("DECANTER SLA ALERT: Details: {}", builder.toString());
- } else {
- LOGGER.warn("DECANTER SLA ALERT: {} out of pattern {}", event.getProperty("alertAttribute"), event.getProperty("alertPattern"));
+ }
+ if (event.getProperty("alertLevel") != null && ((String) event.getProperty("alertLevel")).equalsIgnoreCase("warn")) {
+ if (backToNormal) {
+ LOGGER.info("DECANTER SLA ALERT BACK TO NORMAL: {} was out of pattern {}", event.getProperty("alertAttribute"), event.getProperty("alertPattern"));
+ } else {
+ LOGGER.warn("DECANTER SLA ALERT: {} out of pattern {}", event.getProperty("alertAttribute"), event.getProperty("alertPattern"));
+ }
LOGGER.warn("DECANTER SLA ALERT: Details: {}", builder.toString());
}
}