You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2025/09/04 12:17:46 UTC
(skywalking-banyandb) 01/11: Refactor batch publisher and client health check logic
This is an automated email from the ASF dual-hosted git repository.
hanahmily pushed a commit to branch pub/perf
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 3d162cbbe4c578d1442b472592cf43a2d27bc385
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Thu Sep 4 03:44:32 2025 +0800
Refactor batch publisher and client health check logic
- Updated the batch publisher to pass the current node name to goroutines for better error handling during message publishing.
- Modified the client health check to deduplicate concurrent probes for the same node and topic, improving efficiency and reducing unnecessary checks.
---
banyand/queue/pub/batch.go | 9 +++++----
banyand/queue/pub/client.go | 39 ++++++++++++++++++++++++++++--------
banyand/queue/pub/client_test.go | 2 +-
banyand/queue/pub/pub.go | 43 +++++++++++++++++++++-------------------
4 files changed, 60 insertions(+), 33 deletions(-)
diff --git a/banyand/queue/pub/batch.go b/banyand/queue/pub/batch.go
index 00a442ef..4d6fdc8c 100644
--- a/banyand/queue/pub/batch.go
+++ b/banyand/queue/pub/batch.go
@@ -147,7 +147,8 @@ func (bp *batchPublisher) Publish(ctx context.Context, topic bus.Topic, messages
}
bp.f.events = append(bp.f.events, make(chan batchEvent))
_ = sendData()
- go func(s clusterv1.Service_SendClient, deferFn func(), bc chan batchEvent) {
+ nodeName := node
+ go func(s clusterv1.Service_SendClient, deferFn func(), bc chan batchEvent, curNode string) {
defer func() {
close(bc)
deferFn()
@@ -160,7 +161,7 @@ func (bp *batchPublisher) Publish(ctx context.Context, topic bus.Topic, messages
resp, errRecv := s.Recv()
if errRecv != nil {
if isFailoverError(errRecv) {
- bc <- batchEvent{n: node, e: common.NewErrorWithStatus(modelv1.Status_STATUS_INTERNAL_ERROR, errRecv.Error())}
+ bc <- batchEvent{n: curNode, e: common.NewErrorWithStatus(modelv1.Status_STATUS_INTERNAL_ERROR, errRecv.Error())}
}
return
}
@@ -172,9 +173,9 @@ func (bp *batchPublisher) Publish(ctx context.Context, topic bus.Topic, messages
}
if isFailoverStatus(resp.Status) {
ce := common.NewErrorWithStatus(resp.Status, resp.Error)
- bc <- batchEvent{n: node, e: ce}
+ bc <- batchEvent{n: curNode, e: ce}
}
- }(stream, deferFn, bp.f.events[len(bp.f.events)-1])
+ }(stream, deferFn, bp.f.events[len(bp.f.events)-1], nodeName)
}
return nil, err
}
diff --git a/banyand/queue/pub/client.go b/banyand/queue/pub/client.go
index 39daf2af..a0a68a6e 100644
--- a/banyand/queue/pub/client.go
+++ b/banyand/queue/pub/client.go
@@ -372,7 +372,8 @@ func (p *pub) checkWritable(n string, topic bus.Topic) (bool, *common.Error) {
if !ok {
return false, nil
}
- err := p.checkServiceHealth(topic.String(), node.conn)
+ topicStr := topic.String()
+ err := p.checkServiceHealth(topicStr, node.conn)
if err == nil {
return true, nil
}
@@ -380,25 +381,47 @@ func (p *pub) checkWritable(n string, topic bus.Topic) (bool, *common.Error) {
if !p.closer.AddRunning() {
return false, err
}
- go func() {
+ // Deduplicate concurrent probes for the same node/topic.
+ p.writableProbeMu.Lock()
+ if _, ok := p.writableProbe[n]; !ok {
+ p.writableProbe[n] = make(map[string]struct{})
+ }
+ if _, exists := p.writableProbe[n][topicStr]; exists {
+ p.writableProbeMu.Unlock()
+ return false, err
+ }
+ p.writableProbe[n][topicStr] = struct{}{}
+ p.writableProbeMu.Unlock()
+
+ go func(nodeName, t string) {
defer p.closer.Done()
+ defer func() {
+ p.writableProbeMu.Lock()
+ if topics, ok := p.writableProbe[nodeName]; ok {
+ delete(topics, t)
+ if len(topics) == 0 {
+ delete(p.writableProbe, nodeName)
+ }
+ }
+ p.writableProbeMu.Unlock()
+ }()
backoff := initBackoff
for {
select {
case <-time.After(backoff):
- if errInternal := p.checkServiceHealth(topic.String(), node.conn); errInternal != nil {
+ if errInternal := p.checkServiceHealth(t, node.conn); errInternal == nil {
func() {
p.mu.Lock()
defer p.mu.Unlock()
- node, ok := p.active[n]
- if !ok {
+ nodeCur, okCur := p.active[nodeName]
+ if !okCur {
return
}
- h.OnAddOrUpdate(node.md)
+ h.OnAddOrUpdate(nodeCur.md)
}()
return
}
- p.log.Warn().Str("topic", topic.String()).Err(err).Str("node", n).Dur("backoff", backoff).Msg("data node can not ingest data")
+ p.log.Warn().Str("topic", t).Err(err).Str("node", nodeName).Dur("backoff", backoff).Msg("data node can not ingest data")
case <-p.closer.CloseNotify():
return
}
@@ -408,7 +431,7 @@ func (p *pub) checkWritable(n string, topic bus.Topic) (bool, *common.Error) {
backoff = maxBackoff
}
}
- }()
+ }(n, topicStr)
return false, err
}
diff --git a/banyand/queue/pub/client_test.go b/banyand/queue/pub/client_test.go
index 59ad104d..0b797cb1 100644
--- a/banyand/queue/pub/client_test.go
+++ b/banyand/queue/pub/client_test.go
@@ -144,7 +144,7 @@ var _ = ginkgo.Describe("publish clients register/unregister", func() {
verifyClientsWithGomega(gomega.Default, p, data.TopicMeasureWrite, 1, 0, 1, 0)
hs.SetServingStatus(data.TopicStreamWrite.String(), grpc_health_v1.HealthCheckResponse_SERVING)
gomega.Eventually(func(g gomega.Gomega) {
- verifyClientsWithGomega(g, p, data.TopicStreamWrite, 1, 0, 2, 1)
+ verifyClientsWithGomega(g, p, data.TopicStreamWrite, 1, 0, 1, 1)
}, flags.EventuallyTimeout).Should(gomega.Succeed())
})
diff --git a/banyand/queue/pub/pub.go b/banyand/queue/pub/pub.go
index f5feaf6a..987931db 100644
--- a/banyand/queue/pub/pub.go
+++ b/banyand/queue/pub/pub.go
@@ -63,18 +63,20 @@ var (
type pub struct {
schema.UnimplementedOnInitHandler
- metadata metadata.Repo
- evictable map[string]evictNode
- log *logger.Logger
- registered map[string]*databasev1.Node
- active map[string]*client
- handlers map[bus.Topic]schema.EventHandler
- closer *run.Closer
- caCertPath string
- prefix string
- allowedRoles []databasev1.Role
- mu sync.RWMutex
- tlsEnabled bool
+ metadata metadata.Repo
+ handlers map[bus.Topic]schema.EventHandler
+ log *logger.Logger
+ registered map[string]*databasev1.Node
+ active map[string]*client
+ evictable map[string]evictNode
+ closer *run.Closer
+ writableProbe map[string]map[string]struct{}
+ caCertPath string
+ prefix string
+ allowedRoles []databasev1.Role
+ mu sync.RWMutex
+ writableProbeMu sync.Mutex
+ tlsEnabled bool
}
func (p *pub) FlagSet() *run.FlagSet {
@@ -279,14 +281,15 @@ func New(metadata metadata.Repo, roles ...databasev1.Role) queue.Client {
}
}
p := &pub{
- metadata: metadata,
- active: make(map[string]*client),
- evictable: make(map[string]evictNode),
- registered: make(map[string]*databasev1.Node),
- handlers: make(map[bus.Topic]schema.EventHandler),
- closer: run.NewCloser(1),
- allowedRoles: roles,
- prefix: strBuilder.String(),
+ metadata: metadata,
+ active: make(map[string]*client),
+ evictable: make(map[string]evictNode),
+ registered: make(map[string]*databasev1.Node),
+ handlers: make(map[bus.Topic]schema.EventHandler),
+ closer: run.NewCloser(1),
+ allowedRoles: roles,
+ prefix: strBuilder.String(),
+ writableProbe: make(map[string]map[string]struct{}),
}
return p
}