You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/09/04 12:17:46 UTC

(skywalking-banyandb) 01/11: Refactor batch publisher and client health check logic

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch pub/perf
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit 3d162cbbe4c578d1442b472592cf43a2d27bc385
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Sep 4 03:44:32 2025 +0800

    Refactor batch publisher and client health check logic
    
    - Updated the batch publisher to pass the current node name to goroutines for better error handling during message publishing.
    - Modified the client health check to deduplicate concurrent probes for the same node and topic, improving efficiency and reducing unnecessary checks.
---
 banyand/queue/pub/batch.go       |  9 +++++----
 banyand/queue/pub/client.go      | 39 ++++++++++++++++++++++++++++--------
 banyand/queue/pub/client_test.go |  2 +-
 banyand/queue/pub/pub.go         | 43 +++++++++++++++++++++-------------------
 4 files changed, 60 insertions(+), 33 deletions(-)

diff --git a/banyand/queue/pub/batch.go b/banyand/queue/pub/batch.go
index 00a442ef..4d6fdc8c 100644
--- a/banyand/queue/pub/batch.go
+++ b/banyand/queue/pub/batch.go
@@ -147,7 +147,8 @@ func (bp *batchPublisher) Publish(ctx context.Context, topic bus.Topic, messages
 		}
 		bp.f.events = append(bp.f.events, make(chan batchEvent))
 		_ = sendData()
-		go func(s clusterv1.Service_SendClient, deferFn func(), bc chan batchEvent) {
+		nodeName := node
+		go func(s clusterv1.Service_SendClient, deferFn func(), bc chan batchEvent, curNode string) {
 			defer func() {
 				close(bc)
 				deferFn()
@@ -160,7 +161,7 @@ func (bp *batchPublisher) Publish(ctx context.Context, topic bus.Topic, messages
 			resp, errRecv := s.Recv()
 			if errRecv != nil {
 				if isFailoverError(errRecv) {
-					bc <- batchEvent{n: node, e: common.NewErrorWithStatus(modelv1.Status_STATUS_INTERNAL_ERROR, errRecv.Error())}
+					bc <- batchEvent{n: curNode, e: common.NewErrorWithStatus(modelv1.Status_STATUS_INTERNAL_ERROR, errRecv.Error())}
 				}
 				return
 			}
@@ -172,9 +173,9 @@ func (bp *batchPublisher) Publish(ctx context.Context, topic bus.Topic, messages
 			}
 			if isFailoverStatus(resp.Status) {
 				ce := common.NewErrorWithStatus(resp.Status, resp.Error)
-				bc <- batchEvent{n: node, e: ce}
+				bc <- batchEvent{n: curNode, e: ce}
 			}
-		}(stream, deferFn, bp.f.events[len(bp.f.events)-1])
+		}(stream, deferFn, bp.f.events[len(bp.f.events)-1], nodeName)
 	}
 	return nil, err
 }
diff --git a/banyand/queue/pub/client.go b/banyand/queue/pub/client.go
index 39daf2af..a0a68a6e 100644
--- a/banyand/queue/pub/client.go
+++ b/banyand/queue/pub/client.go
@@ -372,7 +372,8 @@ func (p *pub) checkWritable(n string, topic bus.Topic) (bool, *common.Error) {
 	if !ok {
 		return false, nil
 	}
-	err := p.checkServiceHealth(topic.String(), node.conn)
+	topicStr := topic.String()
+	err := p.checkServiceHealth(topicStr, node.conn)
 	if err == nil {
 		return true, nil
 	}
@@ -380,25 +381,47 @@ func (p *pub) checkWritable(n string, topic bus.Topic) (bool, *common.Error) {
 	if !p.closer.AddRunning() {
 		return false, err
 	}
-	go func() {
+	// Deduplicate concurrent probes for the same node/topic.
+	p.writableProbeMu.Lock()
+	if _, ok := p.writableProbe[n]; !ok {
+		p.writableProbe[n] = make(map[string]struct{})
+	}
+	if _, exists := p.writableProbe[n][topicStr]; exists {
+		p.writableProbeMu.Unlock()
+		return false, err
+	}
+	p.writableProbe[n][topicStr] = struct{}{}
+	p.writableProbeMu.Unlock()
+
+	go func(nodeName, t string) {
 		defer p.closer.Done()
+		defer func() {
+			p.writableProbeMu.Lock()
+			if topics, ok := p.writableProbe[nodeName]; ok {
+				delete(topics, t)
+				if len(topics) == 0 {
+					delete(p.writableProbe, nodeName)
+				}
+			}
+			p.writableProbeMu.Unlock()
+		}()
 		backoff := initBackoff
 		for {
 			select {
 			case <-time.After(backoff):
-				if errInternal := p.checkServiceHealth(topic.String(), node.conn); errInternal != nil {
+				if errInternal := p.checkServiceHealth(t, node.conn); errInternal == nil {
 					func() {
 						p.mu.Lock()
 						defer p.mu.Unlock()
-						node, ok := p.active[n]
-						if !ok {
+						nodeCur, okCur := p.active[nodeName]
+						if !okCur {
 							return
 						}
-						h.OnAddOrUpdate(node.md)
+						h.OnAddOrUpdate(nodeCur.md)
 					}()
 					return
 				}
-				p.log.Warn().Str("topic", topic.String()).Err(err).Str("node", n).Dur("backoff", backoff).Msg("data node can not ingest data")
+				p.log.Warn().Str("topic", t).Err(err).Str("node", nodeName).Dur("backoff", backoff).Msg("data node can not ingest data")
 			case <-p.closer.CloseNotify():
 				return
 			}
@@ -408,7 +431,7 @@ func (p *pub) checkWritable(n string, topic bus.Topic) (bool, *common.Error) {
 				backoff = maxBackoff
 			}
 		}
-	}()
+	}(n, topicStr)
 	return false, err
 }
 
diff --git a/banyand/queue/pub/client_test.go b/banyand/queue/pub/client_test.go
index 59ad104d..0b797cb1 100644
--- a/banyand/queue/pub/client_test.go
+++ b/banyand/queue/pub/client_test.go
@@ -144,7 +144,7 @@ var _ = ginkgo.Describe("publish clients register/unregister", func() {
 		verifyClientsWithGomega(gomega.Default, p, data.TopicMeasureWrite, 1, 0, 1, 0)
 		hs.SetServingStatus(data.TopicStreamWrite.String(), grpc_health_v1.HealthCheckResponse_SERVING)
 		gomega.Eventually(func(g gomega.Gomega) {
-			verifyClientsWithGomega(g, p, data.TopicStreamWrite, 1, 0, 2, 1)
+			verifyClientsWithGomega(g, p, data.TopicStreamWrite, 1, 0, 1, 1)
 		}, flags.EventuallyTimeout).Should(gomega.Succeed())
 	})
 
diff --git a/banyand/queue/pub/pub.go b/banyand/queue/pub/pub.go
index f5feaf6a..987931db 100644
--- a/banyand/queue/pub/pub.go
+++ b/banyand/queue/pub/pub.go
@@ -63,18 +63,20 @@ var (
 
 type pub struct {
 	schema.UnimplementedOnInitHandler
-	metadata     metadata.Repo
-	evictable    map[string]evictNode
-	log          *logger.Logger
-	registered   map[string]*databasev1.Node
-	active       map[string]*client
-	handlers     map[bus.Topic]schema.EventHandler
-	closer       *run.Closer
-	caCertPath   string
-	prefix       string
-	allowedRoles []databasev1.Role
-	mu           sync.RWMutex
-	tlsEnabled   bool
+	metadata        metadata.Repo
+	handlers        map[bus.Topic]schema.EventHandler
+	log             *logger.Logger
+	registered      map[string]*databasev1.Node
+	active          map[string]*client
+	evictable       map[string]evictNode
+	closer          *run.Closer
+	writableProbe   map[string]map[string]struct{}
+	caCertPath      string
+	prefix          string
+	allowedRoles    []databasev1.Role
+	mu              sync.RWMutex
+	writableProbeMu sync.Mutex
+	tlsEnabled      bool
 }
 
 func (p *pub) FlagSet() *run.FlagSet {
@@ -279,14 +281,15 @@ func New(metadata metadata.Repo, roles ...databasev1.Role) queue.Client {
 		}
 	}
 	p := &pub{
-		metadata:     metadata,
-		active:       make(map[string]*client),
-		evictable:    make(map[string]evictNode),
-		registered:   make(map[string]*databasev1.Node),
-		handlers:     make(map[bus.Topic]schema.EventHandler),
-		closer:       run.NewCloser(1),
-		allowedRoles: roles,
-		prefix:       strBuilder.String(),
+		metadata:      metadata,
+		active:        make(map[string]*client),
+		evictable:     make(map[string]evictNode),
+		registered:    make(map[string]*databasev1.Node),
+		handlers:      make(map[bus.Topic]schema.EventHandler),
+		closer:        run.NewCloser(1),
+		allowedRoles:  roles,
+		prefix:        strBuilder.String(),
+		writableProbe: make(map[string]map[string]struct{}),
 	}
 	return p
 }