You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by pe...@apache.org on 2018/12/01 06:18:36 UTC
[incubator-skywalking] branch master updated: Make endpoint and
instance alarm active (#1987)
This is an automated email from the ASF dual-hosted git repository.
pengys pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-skywalking.git
The following commit(s) were added to refs/heads/master by this push:
new 2d8f390 Make endpoint and instance alarm active (#1987)
2d8f390 is described below
commit 2d8f390351a15ea73d4810fb7d2fce74f6bbb123
Author: 吴晟 Wu Sheng <wu...@foxmail.com>
AuthorDate: Sat Dec 1 14:18:31 2018 +0800
Make endpoint and instance alarm active (#1987)
* Support alarm in service instance and endpoint.
* Fix alarm default settings and document.
* Fix wrong default setting
* Make service instance and endpoint scopes available in alarm core.
* Fix alarm bug.
---
docs/en/setup/backend/backend-alarm.md | 11 ++++-
.../oap/server/core/alarm/provider/AlarmCore.java | 12 ++++--
.../server/core/alarm/provider/NotifyHandler.java | 4 ++
.../server/core/alarm/provider/RunningRule.java | 11 ++---
.../oap/server/core/alarm/AlarmEntrance.java | 34 +++++++++++++++-
.../oap/server/core/alarm/EndpointMetaInAlarm.java | 47 ++++++++++++++++++++++
.../core/alarm/ServiceInstanceMetaInAlarm.java | 47 ++++++++++++++++++++++
.../src/main/assembly/alarm-settings.yml | 26 +++++++++---
8 files changed, 173 insertions(+), 19 deletions(-)
diff --git a/docs/en/setup/backend/backend-alarm.md b/docs/en/setup/backend/backend-alarm.md
index c4ff6f5..aa22b9f 100644
--- a/docs/en/setup/backend/backend-alarm.md
+++ b/docs/en/setup/backend/backend-alarm.md
@@ -49,9 +49,18 @@ rules:
count: 4
```
+## Default alarm rules
+We provided a default `alarm-setting.yml` in our distribution only for convenience, which including following rules
+1. Service average response time over 1s in last 3 minutes.
+1. Service success rate lower than 80% in last 2 minutes.
+1. Service 90% response time is lower than 1000ms in last 3 minutes
+1. Service Instance average response time over 1s in last 2 minutes.
+1. Endpoint average response time over 1s in last 2 minutes.
+
+
## List of all potential metric name
The metric names are defined in official [OAL scripts](../../guides/backend-oal-scripts.md), right now
-only metric from **Service** scope could be used in Alarm, we will extend in further versions.
+metric from **Service**, **Service Instance**, **Endpoint** scopes could be used in Alarm, we will extend in further versions.
Submit issue or pull request if you want to support any other scope in alarm.
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java
index e76591d..b43e18c 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmCore.java
@@ -69,21 +69,25 @@ public class AlarmCore {
Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
try {
List<AlarmMessage> alarmMessageList = new ArrayList<>(30);
+ LocalDateTime checkTime = LocalDateTime.now();
+ int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes();
+ boolean[] hasExecute = new boolean[] {false};
runningContext.values().forEach(ruleList -> ruleList.forEach(runningRule -> {
- LocalDateTime checkTime = LocalDateTime.now();
- int minutes = Minutes.minutesBetween(lastExecuteTime, checkTime).getMinutes();
if (minutes > 0) {
+ hasExecute[0] = true;
runningRule.moveTo(checkTime);
/**
* Don't run in the first quarter per min, avoid to trigger false alarm.
*/
if (checkTime.getSecondOfMinute() > 15) {
alarmMessageList.addAll(runningRule.check());
- // Set the last execute time, and make sure the second is `00`, such as: 18:30:00
- lastExecuteTime = checkTime.minusSeconds(checkTime.getSecondOfMinute());
}
}
}));
+ // Set the last execute time, and make sure the second is `00`, such as: 18:30:00
+ if (hasExecute[0]) {
+ lastExecuteTime = checkTime.minusSeconds(checkTime.getSecondOfMinute());
+ }
if (alarmMessageList.size() > 0) {
allCallbacks.forEach(callback -> callback.doAlarm(alarmMessageList));
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
index 1d4aacc..408f9ee 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/NotifyHandler.java
@@ -38,6 +38,10 @@ public class NotifyHandler implements IndicatorNotify {
switch (meta.getScope()) {
case Service:
break;
+ case ServiceInstance:
+ break;
+ case Endpoint:
+ break;
default:
return;
}
diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java
index a010392..dd0659d 100644
--- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java
+++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java
@@ -118,14 +118,9 @@ public class RunningRule {
Window window = windows.get(meta);
if (window == null) {
window = new Window(period);
- Window ifAbsent = windows.putIfAbsent(meta, window);
- if (ifAbsent == null) {
- LocalDateTime timebucket = TIME_BUCKET_FORMATTER.parseLocalDateTime(indicator.getTimeBucket() + "");
- window.moveTo(timebucket);
- } else {
- window = windows.get(meta);
- }
-
+ LocalDateTime timebucket = TIME_BUCKET_FORMATTER.parseLocalDateTime(indicator.getTimeBucket() + "");
+ window.moveTo(timebucket);
+ windows.put(meta, window);
}
window.add(indicator);
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java
index 3992841..b2d0e41 100644
--- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmEntrance.java
@@ -21,7 +21,11 @@ package org.apache.skywalking.oap.server.core.alarm;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.indicator.Indicator;
+import org.apache.skywalking.oap.server.core.cache.EndpointInventoryCache;
+import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
+import org.apache.skywalking.oap.server.core.register.EndpointInventory;
+import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
import org.apache.skywalking.oap.server.core.register.ServiceInventory;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
@@ -31,6 +35,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleManager;
public class AlarmEntrance {
private ModuleManager moduleManager;
private ServiceInventoryCache serviceInventoryCache;
+ private ServiceInstanceInventoryCache serviceInstanceInventoryCache;
+ private EndpointInventoryCache endpointInventoryCache;
private IndicatorNotify indicatorNotify;
private ReentrantLock initLock;
@@ -48,7 +54,7 @@ public class AlarmEntrance {
AlarmMeta alarmMeta = ((AlarmSupported)indicator).getAlarmMeta();
- MetaInAlarm metaInAlarm = null;
+ MetaInAlarm metaInAlarm;
switch (alarmMeta.getScope()) {
case Service:
int serviceId = Integer.parseInt(alarmMeta.getId());
@@ -59,6 +65,30 @@ public class AlarmEntrance {
serviceMetaInAlarm.setName(serviceInventory.getName());
metaInAlarm = serviceMetaInAlarm;
break;
+ case ServiceInstance:
+ int serviceInstanceId = Integer.parseInt(alarmMeta.getId());
+ ServiceInstanceInventory serviceInstanceInventory = serviceInstanceInventoryCache.get(serviceInstanceId);
+ ServiceInstanceMetaInAlarm instanceMetaInAlarm = new ServiceInstanceMetaInAlarm();
+ instanceMetaInAlarm.setIndicatorName(alarmMeta.getIndicatorName());
+ instanceMetaInAlarm.setId(serviceInstanceId);
+ instanceMetaInAlarm.setName(serviceInstanceInventory.getName());
+ metaInAlarm = instanceMetaInAlarm;
+ break;
+ case Endpoint:
+ int endpointId = Integer.parseInt(alarmMeta.getId());
+ EndpointInventory endpointInventory = endpointInventoryCache.get(endpointId);
+ EndpointMetaInAlarm endpointMetaInAlarm = new EndpointMetaInAlarm();
+ endpointMetaInAlarm.setIndicatorName(alarmMeta.getIndicatorName());
+ endpointMetaInAlarm.setId(endpointId);
+
+ serviceId = endpointInventory.getServiceId();
+ serviceInventory = serviceInventoryCache.get(serviceId);
+
+ String textName = endpointInventory.getName() + " in " + serviceInventory.getName();
+
+ endpointMetaInAlarm.setName(textName);
+ metaInAlarm = endpointMetaInAlarm;
+ break;
default:
return;
}
@@ -72,6 +102,8 @@ public class AlarmEntrance {
try {
if (serviceInventoryCache == null) {
serviceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class);
+ serviceInstanceInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInstanceInventoryCache.class);
+ endpointInventoryCache = moduleManager.find(CoreModule.NAME).provider().getService(EndpointInventoryCache.class);
indicatorNotify = moduleManager.find(AlarmModule.NAME).provider().getService(IndicatorNotify.class);
indicatorNotify.init(new AlarmStandardPersistence());
}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/EndpointMetaInAlarm.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/EndpointMetaInAlarm.java
new file mode 100644
index 0000000..10b859f
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/EndpointMetaInAlarm.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.alarm;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.source.Scope;
+
+@Getter(AccessLevel.PUBLIC)
+@Setter(AccessLevel.PUBLIC)
+public class EndpointMetaInAlarm extends MetaInAlarm {
+ private String indicatorName;
+
+ private int id;
+ private String name;
+ private String[] tags;
+ private String[] properties;
+
+ @Override public Scope getScope() {
+ return Scope.Endpoint;
+ }
+
+ @Override public int getId0() {
+ return id;
+ }
+
+ @Override public int getId1() {
+ return 0;
+ }
+}
diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/ServiceInstanceMetaInAlarm.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/ServiceInstanceMetaInAlarm.java
new file mode 100644
index 0000000..4c8862a
--- /dev/null
+++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/ServiceInstanceMetaInAlarm.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.alarm;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.source.Scope;
+
+@Getter(AccessLevel.PUBLIC)
+@Setter(AccessLevel.PUBLIC)
+public class ServiceInstanceMetaInAlarm extends MetaInAlarm {
+ private String indicatorName;
+
+ private int id;
+ private String name;
+ private String[] tags;
+ private String[] properties;
+
+ @Override public Scope getScope() {
+ return Scope.ServiceInstance;
+ }
+
+ @Override public int getId0() {
+ return id;
+ }
+
+ @Override public int getId1() {
+ return 0;
+ }
+}
diff --git a/oap-server/server-starter/src/main/assembly/alarm-settings.yml b/oap-server/server-starter/src/main/assembly/alarm-settings.yml
index 5b839fb..9ee76e2 100644
--- a/oap-server/server-starter/src/main/assembly/alarm-settings.yml
+++ b/oap-server/server-starter/src/main/assembly/alarm-settings.yml
@@ -24,28 +24,44 @@ rules:
period: 10
count: 3
silence-period: 5
- message: Response time of service {name} is more than 2000ms.
+ message: Response time of service {name} is more than 1000ms in last 3 minutes.
service_sla_rule:
# Indicator value need to be long, double or int
indicator-name: service_sla
op: "<"
- threshold: 80
+ threshold: 8000
# The length of time to evaluate the metric
period: 10
# How many times after the metric match the condition, will trigger alarm
count: 2
# How many times of checks, the alarm keeps silence after alarm triggered, default as same as period.
silence-period: 3
- message: Successful rate of service {name} is lower than 80%
+ message: Successful rate of service {name} is lower than 80% in last 2 minutes.
service_p90_sla_rule:
# Indicator value need to be long, double or int
- indicator-name: service_sla
+ indicator-name: service_p90
op: ">"
threshold: 1000
period: 10
count: 3
silence-period: 5
- message: 90% response time of service {name} is lower than 80%
+ message: 90% response time of service {name} is lower than 1000ms in last 3 minutes
+ service_instance_resp_time_rule:
+ indicator-name: service_instance_resp_time
+ op: ">"
+ threshold: 1000
+ period: 10
+ count: 2
+ silence-period: 5
+ message: Response time of service instance {name} is more than 1000ms in last 2 minutes.
+ endpoint_avg_rule:
+ indicator-name: endpoint_avg
+ op: ">"
+ threshold: 1000
+ period: 10
+ count: 2
+ silence-period: 5
+ message: Response time of endpoint {name} is more than 1000ms in last 2 minutes.
webhooks:
# - http://127.0.0.1/notify/