You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dubbo.apache.org by zh...@apache.org on 2024/09/18 02:32:58 UTC
(dubbo-kubernetes) branch master updated: [horus] Optimize alarm logs (#352)
This is an automated email from the ASF dual-hosted git repository.
zhongxjian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/dubbo-kubernetes.git
The following commit(s) were added to refs/heads/master by this push:
new 7f288da2 [horus] Optimize alarm logs (#352)
7f288da2 is described below
commit 7f288da2dc010b4492451a8ce1957e4b3403ab76
Author: mfordjody <11...@qq.com>
AuthorDate: Wed Sep 18 10:32:53 2024 +0800
[horus] Optimize alarm logs (#352)
---
app/horus/core/alert/dingtalk.go | 3 +--
app/horus/core/horuser/modular.go | 4 ++--
app/horus/core/horuser/recovery.go | 3 +--
deploy/horus/horus.yaml | 9 ++-------
4 files changed, 6 insertions(+), 13 deletions(-)
diff --git a/app/horus/core/alert/dingtalk.go b/app/horus/core/alert/dingtalk.go
index 94c6e4a8..d012d604 100644
--- a/app/horus/core/alert/dingtalk.go
+++ b/app/horus/core/alert/dingtalk.go
@@ -54,8 +54,7 @@ type Message struct {
func DingTalkSend(dk *config.DingTalkConfiguration, msg string) {
dtm := Message{MsgType: "text"}
- dtm.Text.Content = fmt.Sprintf("%s\n"+
- "日志：%s】", DingTalkTitle, msg)
+ dtm.Text.Content = fmt.Sprint(DingTalkTitle, msg)
dtm.At.AtMobiles = dk.AtMobiles
bs, err := json.Marshal(dtm)
if err != nil {
diff --git a/app/horus/core/horuser/modular.go b/app/horus/core/horuser/modular.go
index 1a77a552..45943829 100644
--- a/app/horus/core/horuser/modular.go
+++ b/app/horus/core/horuser/modular.go
@@ -91,7 +91,7 @@ func (h *Horuser) CustomizeModularNodes(clusterName, moduleName, nodeName, ip st
dailyLimit := h.cc.CustomModular.CordonDailyLimit[moduleName]
if len(data) > dailyLimit {
- msg := fmt.Sprintf("【日期:%v】 【集群:%v\n】 【模块今日 Cordon 节点数: %v】\n 【已达到今日上限: %v】\n 【节点:%v】",
+ msg := fmt.Sprintf("【日期:%v】 【集群:%v\n】 【今日 Cordon 节点数: %v】\n 【已达到今日上限: %v】\n 【节点:%v】",
data, clusterName, moduleName, dailyLimit, nodeName)
alert.DingTalkSend(h.cc.CustomModular.DingTalk, msg)
return
@@ -118,7 +118,7 @@ func (h *Horuser) CustomizeModularNodes(clusterName, moduleName, nodeName, ip st
klog.Errorf("Cordon failed:%v", res)
}
- msg := fmt.Sprintf("【集群:%v】\n 【%s 插件 Cordon 节点:%v】\n 【结果: %v】\n 【今日操作次数:%v",
+ msg := fmt.Sprintf("\n【集群:%v】\n【发现 %s 异常已禁止调度】\n【已禁止调度节点:%v】\n 【处理结果: %v】\n 【今日操作次数:%v】\n",
clusterName, moduleName, nodeName, res, len(data)+1)
klog.Infof("Attempting to send DingTalk message: %s", msg)
diff --git a/app/horus/core/horuser/recovery.go b/app/horus/core/horuser/recovery.go
index 515cf2d8..dd59c5c4 100644
--- a/app/horus/core/horuser/recovery.go
+++ b/app/horus/core/horuser/recovery.go
@@ -80,8 +80,7 @@ func (h *Horuser) recoveryNodes(n *db.NodeDataInfo) {
if err != nil {
res = fmt.Sprintf("failed:%v", err)
}
- msg := fmt.Sprintf("【自愈检查 %v: 恢复节点调度】【集群: %v】\n【节点: %v】【日期: %v】\n"+
- "【自愈检查 QL: %v", res, n.ClusterName, n.NodeName, n.CreateTime, n.RecoveryQL)
+ msg := fmt.Sprintf("\n【集群: %v】\n【异常节点恢复调度】\n【已恢复调度节点: %v】\n【处理结果：%v】\n【日期: %v】\n", n.ClusterName, n.NodeName, res, n.CreateTime)
alert.DingTalkSend(h.cc.NodeRecovery.DingTalk, msg)
pass, err := n.RecoveryMarker()
diff --git a/deploy/horus/horus.yaml b/deploy/horus/horus.yaml
index 6884df57..768184fd 100644
--- a/deploy/horus/horus.yaml
+++ b/deploy/horus/horus.yaml
@@ -36,7 +36,7 @@ kubeMultiple:
cluster: config.1
promMultiple:
- cluster: http://192.168.15.128:30201
+ cluster: http://192.168.15.128:31160
nodeRecovery:
enabled: true
@@ -52,18 +52,13 @@ nodeRecovery:
customModular:
enabled: true
cordonDailyLimit:
- filesystem_readonly: 5
- arp_entries: 5
+ filesystem_readonly: 1
checkQL:
filesystem_readonly: |-
node_filesystem_readonly{mountpoint="/"} != 1
- arp_entries: |-
- node_arp_entries{device="ens160"} > 2
recoveryQL:
filesystem_readonly: |-
node_filesystem_readonly{mountpoint="/",node="%s"} == 0
- arp_entries: |-
- node_arp_entries{device="ens160",node="%s"} > 2
checkIntervalSecond: 5
promQueryTimeSecond: 60
kubeMultiple: