You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by ti...@apache.org on 2020/09/04 09:16:33 UTC

[servicecomb-service-center] branch master updated: Fix health check bug (#685)

This is an automated email from the ASF dual-hosted git repository.

tianxiaoliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-service-center.git


The following commit(s) were added to refs/heads/master by this push:
     new 2f25dc8  Fix health check bug (#685)
2f25dc8 is described below

commit 2f25dc83abf6f8705d77b5091e6a29d7d7435dcd
Author: humingcheng <hu...@163.com>
AuthorDate: Fri Sep 4 17:13:52 2020 +0800

    Fix health check bug (#685)
---
 server/plugin/registry/etcd/common.go    |  3 +-
 server/plugin/registry/etcd/etcd.go      | 47 +++++++++++++++++---------------
 server/plugin/registry/etcd/etcd_test.go |  1 -
 3 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/server/plugin/registry/etcd/common.go b/server/plugin/registry/etcd/common.go
index 12e6b59..b296c1f 100644
--- a/server/plugin/registry/etcd/common.go
+++ b/server/plugin/registry/etcd/common.go
@@ -23,7 +23,8 @@ import (
 const (
 	// here will new an etcd connection after about 30s(=5s * 3 + (backoff:8s))
 	// when the connected etcd member was hung but tcp is still alive
-	healthCheckTimeout = 5 * time.Second
+	healthCheckTimeout    = 5 * time.Second
+	healthCheckRetryTimes = 3
 
 	// see google.golang.org/grpc/keepalive/keepalive.go
 	// after a duration of this time if the client doesn't see any activity
diff --git a/server/plugin/registry/etcd/etcd.go b/server/plugin/registry/etcd/etcd.go
index 6251e35..159204f 100644
--- a/server/plugin/registry/etcd/etcd.go
+++ b/server/plugin/registry/etcd/etcd.go
@@ -723,40 +723,43 @@ func (c *Client) HealthCheck() {
 }
 
 func (c *Client) healthCheckLoop(pctx context.Context) {
-	retries, start := 0, time.Now()
 	d := c.AutoSyncInterval
 	for {
+		var healthCheckErr error
 		select {
 		case <-pctx.Done():
 			return
 		case <-time.After(d):
-			var err error
-			ctx, cancel := context.WithTimeout(c.Client.Ctx(), healthCheckTimeout)
-			defer cancel()
-			if err = c.SyncMembers(ctx); err != nil {
-				d := backoff.GetBackoff().Delay(retries)
-				retries++
-				log.Errorf(err, "retry to sync members from etcd %s after %s", c.Endpoints, d)
+			for i := 0; i < healthCheckRetryTimes; i++ {
+				ctx, cancel := context.WithTimeout(c.Client.Ctx(), healthCheckTimeout)
+				healthCheckErr = c.SyncMembers(ctx)
+				cancel()
+				if healthCheckErr == nil {
+					break
+				}
+				d := backoff.GetBackoff().Delay(i)
+				log.Errorf(healthCheckErr, "retry to sync members from etcd %s after %s", c.Endpoints, d)
 				select {
 				case <-pctx.Done():
 					return
-				default:
-					continue
-				}
-			} else {
-				log.Info("sync members ok.")
-				if err := alarm.Clear(alarm.IDBackendConnectionRefuse); err != nil {
-					log.Error("", err)
-				}
-				if cerr := c.ReOpen(); cerr != nil {
-					log.Errorf(cerr, "retry to health check etcd %s after %s", c.Endpoints, c.AutoSyncInterval)
-				} else {
-					log.Infof("[%s]re-connected to etcd %s", time.Since(start), c.Endpoints)
-					continue
+				case <-time.After(d):
 				}
-				return
 			}
+		}
+
+		var alarmErr error
+		if healthCheckErr != nil {
+			log.Error("etcd health check failed", healthCheckErr)
+			alarmErr = alarm.Raise(alarm.IDBackendConnectionRefuse, alarm.AdditionalContext(healthCheckErr.Error()))
+			if err := c.ReOpen(); err != nil {
+				log.Error("re-connect to etcd failed", err)
+			}
+		} else {
+			alarmErr = alarm.Clear(alarm.IDBackendConnectionRefuse)
+		}
 
+		if alarmErr != nil {
+			log.Error("alarm failed", alarmErr)
 		}
 	}
 }
diff --git a/server/plugin/registry/etcd/etcd_test.go b/server/plugin/registry/etcd/etcd_test.go
index 8edc293..bfdee00 100644
--- a/server/plugin/registry/etcd/etcd_test.go
+++ b/server/plugin/registry/etcd/etcd_test.go
@@ -565,7 +565,6 @@ func TestEtcdClient_HealthCheck(t *testing.T) {
 
 	etcdc.Endpoints = []string{endpoint}
 
-	etcdc.Close()
 	ctx, _ = context.WithTimeout(context.Background(), 1*time.Second)
 	go etcdc.healthCheckLoop(ctx)
 	for {