You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by wu...@apache.org on 2020/08/22 09:18:31 UTC
[skywalking] branch master updated: support Slack alarm hooks
(#5364)
This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 3dfa40e support Slack alarm hooks (#5364)
3dfa40e is described below
commit 3dfa40ebb59da7624d495938ffb2795e11f8b61b
Author: Jared Tan <ji...@daocloud.io>
AuthorDate: Sat Aug 22 17:18:09 2020 +0800
support Slack alarm hooks (#5364)
---
docs/en/setup/backend/backend-alarm.md | 18 +++++++
.../core/alarm/provider/AlarmRulesWatcher.java | 5 ++
.../server/core/alarm/provider/NotifyHandler.java | 2 +
.../oap/server/core/alarm/provider/Rules.java | 2 +
.../server/core/alarm/provider/RulesReader.java | 28 ++++++++--
.../core/alarm/provider/WebhookCallback.java | 15 +++---
.../{Rules.java => slack/SlackSettings.java} | 22 ++++----
.../SlackhookCallback.java} | 61 ++++++++++++++--------
.../src/main/resources/alarm-settings.yml | 12 +++++
9 files changed, 120 insertions(+), 45 deletions(-)
diff --git a/docs/en/setup/backend/backend-alarm.md b/docs/en/setup/backend/backend-alarm.md
index 37317be..90189be 100644
--- a/docs/en/setup/backend/backend-alarm.md
+++ b/docs/en/setup/backend/backend-alarm.md
@@ -162,6 +162,24 @@ message AlarmMessage {
}
```
+## Slack Chat Hook
+To do this you need to follow the [Getting Started with Incoming Webhooks guide](https://api.slack.com/messaging/webhooks) and create new Webhooks.
+
+The alarm message will send through HTTP post by `application/json` content type if you configured Slack Incoming Webhooks as following:
+```yml
+slackHooks:
+ textTemplate: |-
+ {
+ "type": "section",
+ "text": {
+ "type": "mrkdwn",
+ "text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**."
+ }
+ }
+ webhooks:
+ - https://hooks.slack.com/services/x/y/z
+```
+
## Update the settings dynamically
Since 6.5.0, the alarm settings can be updated dynamically at runtime by [Dynamic Configuration](dynamic-config.md),
which will override the settings in `alarm-settings.yml`.
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java
index af4f3ae..51a5353 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRulesWatcher.java
@@ -29,6 +29,7 @@ import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
import org.apache.skywalking.oap.server.core.Const;
import org.apache.skywalking.oap.server.core.alarm.AlarmModule;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
import org.apache.skywalking.oap.server.library.module.ModuleProvider;
/**
@@ -109,4 +110,8 @@ public class AlarmRulesWatcher extends ConfigChangeWatcher {
public GRPCAlarmSetting getGrpchookSetting() {
return this.rules.getGrpchookSetting();
}
+
+ public SlackSettings getSlackSettings() {
+ return this.rules.getSlacks();
+ }
}
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
index bf77c38..2209d19 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
@@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.alarm.MetricsNotify;
import org.apache.skywalking.oap.server.core.alarm.ServiceInstanceMetaInAlarm;
import org.apache.skywalking.oap.server.core.alarm.ServiceMetaInAlarm;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCCallback;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackhookCallback;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics;
import org.apache.skywalking.oap.server.core.analysis.metrics.MetricsMetaInfo;
@@ -158,6 +159,7 @@ public class NotifyHandler implements MetricsNotify {
List<AlarmCallback> allCallbacks = new ArrayList<>(Arrays.asList(callbacks));
allCallbacks.add(new WebhookCallback(alarmRulesWatcher));
allCallbacks.add(new GRPCCallback(alarmRulesWatcher));
+ allCallbacks.add(new SlackhookCallback(alarmRulesWatcher));
core.start(allCallbacks);
}
}
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java
index 87dd507..0f95820 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java
@@ -24,6 +24,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
@Setter
@Getter
@@ -32,6 +33,7 @@ public class Rules {
private List<AlarmRule> rules;
private List<String> webhooks;
private GRPCAlarmSetting grpchookSetting;
+ private SlackSettings slacks;
public Rules() {
this.rules = new ArrayList<>();
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java
index e1c1adf..9f5f947 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RulesReader.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
+import org.apache.skywalking.oap.server.core.alarm.provider.slack.SlackSettings;
import org.yaml.snakeyaml.Yaml;
/**
@@ -65,8 +66,10 @@ public class RulesReader {
alarmRule.setExcludeNames((ArrayList) settings.getOrDefault("exclude-names", new ArrayList(0)));
alarmRule.setIncludeNamesRegex((String) settings.getOrDefault("include-names-regex", ""));
alarmRule.setExcludeNamesRegex((String) settings.getOrDefault("exclude-names-regex", ""));
- alarmRule.setIncludeLabels((ArrayList) settings.getOrDefault("include-labels", new ArrayList(0)));
- alarmRule.setExcludeLabels((ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0)));
+ alarmRule.setIncludeLabels(
+ (ArrayList) settings.getOrDefault("include-labels", new ArrayList(0)));
+ alarmRule.setExcludeLabels(
+ (ArrayList) settings.getOrDefault("exclude-labels", new ArrayList(0)));
alarmRule.setIncludeLabelsRegex((String) settings.getOrDefault("include-labels-regex", ""));
alarmRule.setExcludeLabelsRegex((String) settings.getOrDefault("exclude-labels-regex", ""));
alarmRule.setThreshold(settings.get("threshold").toString());
@@ -74,8 +77,9 @@ public class RulesReader {
alarmRule.setPeriod((Integer) settings.getOrDefault("period", 1));
alarmRule.setCount((Integer) settings.getOrDefault("count", 1));
alarmRule.setSilencePeriod((Integer) settings.getOrDefault("silence-period", -1));
- alarmRule.setMessage((String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule
- .getAlarmRuleName()));
+ alarmRule.setMessage(
+ (String) settings.getOrDefault("message", "Alarm caused by Rule " + alarmRule
+ .getAlarmRuleName()));
rules.getRules().add(alarmRule);
}
@@ -104,6 +108,22 @@ public class RulesReader {
rules.setGrpchookSetting(grpcAlarmSetting);
}
+
+ Map slacks = (Map) yamlData.get("slackHooks");
+ if (slacks != null) {
+ SlackSettings slackSettings = new SlackSettings();
+ Object textTemplate = slacks.getOrDefault("textTemplate", "");
+ slackSettings.setTextTemplate((String) textTemplate);
+
+ List<String> slackWebhooks = (List<String>) slacks.get("webhooks");
+ if (slackWebhooks != null) {
+ slackWebhooks.forEach(
+ url -> slackSettings.getWebhooks().add(url)
+ );
+ }
+
+ rules.setSlacks(slackSettings);
+ }
}
return rules;
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java
index a9d5333..c997d6a 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
@@ -35,14 +36,12 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Use SkyWalking alarm webhook API call a remote endpoints.
*/
+@Slf4j
public class WebhookCallback implements AlarmCallback {
- private static final Logger LOGGER = LoggerFactory.getLogger(WebhookCallback.class);
private static final int HTTP_CONNECT_TIMEOUT = 1000;
private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
private static final int HTTP_SOCKET_TIMEOUT = 10000;
@@ -62,7 +61,7 @@ public class WebhookCallback implements AlarmCallback {
@Override
public void doAlarm(List<AlarmMessage> alarmMessage) {
- if (alarmRulesWatcher.getWebHooks().size() == 0) {
+ if (alarmRulesWatcher.getWebHooks().isEmpty()) {
return;
}
@@ -81,19 +80,19 @@ public class WebhookCallback implements AlarmCallback {
CloseableHttpResponse httpResponse = httpClient.execute(post);
StatusLine statusLine = httpResponse.getStatusLine();
if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) {
- LOGGER.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
+ log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
}
} catch (UnsupportedEncodingException e) {
- LOGGER.error("Alarm to JSON error, " + e.getMessage(), e);
+ log.error("Alarm to JSON error, " + e.getMessage(), e);
} catch (IOException e) {
- LOGGER.error("send alarm to " + url + " failure.", e);
+ log.error("send alarm to " + url + " failure.", e);
}
});
} finally {
try {
httpClient.close();
} catch (IOException e) {
- LOGGER.error(e.getMessage(), e);
+ log.error(e.getMessage(), e);
}
}
}
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java
similarity index 70%
copy from oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java
copy to oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java
index 87dd507..9c3df1c 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/Rules.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackSettings.java
@@ -16,25 +16,27 @@
*
*/
-package org.apache.skywalking.oap.server.core.alarm.provider;
+package org.apache.skywalking.oap.server.core.alarm.provider.slack;
import java.util.ArrayList;
import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
-import org.apache.skywalking.oap.server.core.alarm.provider.grpc.GRPCAlarmSetting;
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
@Setter
@Getter
@ToString
-public class Rules {
- private List<AlarmRule> rules;
- private List<String> webhooks;
- private GRPCAlarmSetting grpchookSetting;
+public class SlackSettings {
- public Rules() {
- this.rules = new ArrayList<>();
- this.webhooks = new ArrayList<>();
- }
+ private String textTemplate;
+
+ @Builder.Default
+ private List<String> webhooks = new ArrayList<>();
}
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java
similarity index 58%
copy from oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java
copy to oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java
index a9d5333..f7eb93e 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/WebhookCallback.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/slack/SlackhookCallback.java
@@ -16,59 +16,60 @@
*
*/
-package org.apache.skywalking.oap.server.core.alarm.provider;
+package org.apache.skywalking.oap.server.core.alarm.provider.slack;
import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
import io.netty.handler.codec.http.HttpHeaderValues;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
import java.util.List;
+import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.skywalking.oap.server.core.alarm.AlarmCallback;
import org.apache.skywalking.oap.server.core.alarm.AlarmMessage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.skywalking.oap.server.core.alarm.provider.AlarmRulesWatcher;
/**
- * Use SkyWalking alarm webhook API call a remote endpoints.
+ * Use SkyWalking alarm slack webhook API call a remote endpoints.
*/
-public class WebhookCallback implements AlarmCallback {
- private static final Logger LOGGER = LoggerFactory.getLogger(WebhookCallback.class);
+@Slf4j
+public class SlackhookCallback implements AlarmCallback {
private static final int HTTP_CONNECT_TIMEOUT = 1000;
private static final int HTTP_CONNECTION_REQUEST_TIMEOUT = 1000;
private static final int HTTP_SOCKET_TIMEOUT = 10000;
-
+ private static final Gson GSON = new Gson();
private AlarmRulesWatcher alarmRulesWatcher;
private RequestConfig requestConfig;
- private Gson gson = new Gson();
- public WebhookCallback(AlarmRulesWatcher alarmRulesWatcher) {
+ public SlackhookCallback(final AlarmRulesWatcher alarmRulesWatcher) {
this.alarmRulesWatcher = alarmRulesWatcher;
- requestConfig = RequestConfig.custom()
- .setConnectTimeout(HTTP_CONNECT_TIMEOUT)
- .setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT)
- .setSocketTimeout(HTTP_SOCKET_TIMEOUT)
- .build();
+ this.requestConfig = RequestConfig.custom()
+ .setConnectTimeout(HTTP_CONNECT_TIMEOUT)
+ .setConnectionRequestTimeout(HTTP_CONNECTION_REQUEST_TIMEOUT)
+ .setSocketTimeout(HTTP_SOCKET_TIMEOUT)
+ .build();
}
@Override
- public void doAlarm(List<AlarmMessage> alarmMessage) {
- if (alarmRulesWatcher.getWebHooks().size() == 0) {
+ public void doAlarm(List<AlarmMessage> alarmMessages) {
+ if (this.alarmRulesWatcher.getSlackSettings().getWebhooks().isEmpty()) {
return;
}
CloseableHttpClient httpClient = HttpClients.custom().build();
try {
- alarmRulesWatcher.getWebHooks().forEach(url -> {
+ this.alarmRulesWatcher.getSlackSettings().getWebhooks().forEach(url -> {
HttpPost post = new HttpPost(url);
post.setConfig(requestConfig);
post.setHeader(HttpHeaders.ACCEPT, HttpHeaderValues.APPLICATION_JSON.toString());
@@ -76,24 +77,38 @@ public class WebhookCallback implements AlarmCallback {
StringEntity entity;
try {
- entity = new StringEntity(gson.toJson(alarmMessage), StandardCharsets.UTF_8);
+
+ JsonObject jsonObject = new JsonObject();
+ JsonArray jsonElements = new JsonArray();
+
+ alarmMessages.forEach(item -> {
+ jsonElements.add(GSON.fromJson(
+ String.format(
+ this.alarmRulesWatcher.getSlackSettings().getTextTemplate(), item.getAlarmMessage()
+ ), JsonObject.class));
+ });
+
+ jsonObject.add("blocks", jsonElements);
+
+ entity = new StringEntity(GSON.toJson(jsonObject), ContentType.APPLICATION_JSON);
+
post.setEntity(entity);
CloseableHttpResponse httpResponse = httpClient.execute(post);
StatusLine statusLine = httpResponse.getStatusLine();
if (statusLine != null && statusLine.getStatusCode() != HttpStatus.SC_OK) {
- LOGGER.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
+ log.error("send alarm to " + url + " failure. Response code: " + statusLine.getStatusCode());
}
} catch (UnsupportedEncodingException e) {
- LOGGER.error("Alarm to JSON error, " + e.getMessage(), e);
+ log.error("Alarm to JSON error, " + e.getMessage(), e);
} catch (IOException e) {
- LOGGER.error("send alarm to " + url + " failure.", e);
+ log.error("send alarm to " + url + " failure.", e);
}
});
} finally {
try {
httpClient.close();
} catch (IOException e) {
- LOGGER.error(e.getMessage(), e);
+ log.error(e.getMessage(), e);
}
}
}
diff --git a/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml b/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml
index c161092..771bc2a 100755
--- a/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml
+++ b/oap-server/server-bootstrap/src/main/resources/alarm-settings.yml
@@ -47,3 +47,15 @@ gRPCHook:
# target_host: 127.0.0.1
# target_port: 9888
+slackHooks:
+ textTemplate: |-
+ {
+ "type": "section",
+ "text": {
+ "type": "mrkdwn",
+ "text": ":alarm_clock: *Apache Skywalking Alarm* \n **%s**."
+ }
+ }
+ webhooks:
+# - https://hooks.slack.com/services/x/y/zssss
+