You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@skywalking.apache.org by ha...@apache.org on 2023/09/19 12:11:11 UTC

[skywalking-banyandb] 02/03: Refactor node selecting process

This is an automated email from the ASF dual-hosted git repository.

hanahmily pushed a commit to branch test
in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git

commit fe2687b975db70eeb7010e33b81ba00045c61829
Author: Gao Hongtao <ha...@gmail.com>
AuthorDate: Mon Sep 18 12:49:48 2023 +0000

    Refactor node selecting process
    
    Signed-off-by: Gao Hongtao <ha...@gmail.com>
---
 banyand/liaison/grpc/discovery.go     | 103 ++++++--------------------------
 banyand/liaison/grpc/measure.go       |  13 +++-
 banyand/liaison/grpc/node.go          | 109 ++++++++++++++++++++++++++++++++++
 banyand/liaison/grpc/registry_test.go |   2 +-
 banyand/liaison/grpc/server.go        |  10 ++--
 banyand/liaison/grpc/stream.go        |  11 +++-
 banyand/metadata/allocator.go         |  78 ------------------------
 banyand/metadata/client.go            |  15 +----
 banyand/metadata/metadata.go          |   1 -
 banyand/metadata/schema/checker.go    |   7 ---
 banyand/metadata/schema/kind.go       |   9 +--
 banyand/metadata/schema/schema.go     |  12 ----
 banyand/metadata/schema/shard.go      |  74 -----------------------
 banyand/queue/local.go                |   4 ++
 banyand/queue/pub/client.go           |   6 ++
 banyand/queue/pub/pub.go              |   8 ++-
 banyand/queue/queue.go                |   2 +
 pkg/cmdsetup/liaison.go               |   2 +-
 pkg/cmdsetup/standalone.go            |   2 +-
 19 files changed, 175 insertions(+), 293 deletions(-)

diff --git a/banyand/liaison/grpc/discovery.go b/banyand/liaison/grpc/discovery.go
index 823183b1..b133d21d 100644
--- a/banyand/liaison/grpc/discovery.go
+++ b/banyand/liaison/grpc/discovery.go
@@ -18,10 +18,8 @@
 package grpc
 
 import (
-	"context"
 	"fmt"
 	"sync"
-	"time"
 
 	"github.com/pkg/errors"
 
@@ -42,13 +40,14 @@ var errNotExist = errors.New("the object doesn't exist")
 type discoveryService struct {
 	pipeline     queue.Client
 	metadataRepo metadata.Repo
+	nodeRegistry NodeRegistry
 	shardRepo    *shardRepo
 	entityRepo   *entityRepo
 	log          *logger.Logger
 	kind         schema.Kind
 }
 
-func newDiscoveryService(pipeline queue.Client, kind schema.Kind, metadataRepo metadata.Repo) *discoveryService {
+func newDiscoveryService(pipeline queue.Client, kind schema.Kind, metadataRepo metadata.Repo, nodeRegistry NodeRegistry) *discoveryService {
 	sr := &shardRepo{shardEventsMap: make(map[identity]uint32)}
 	er := &entityRepo{entitiesMap: make(map[identity]partition.EntityLocator)}
 	return &discoveryService{
@@ -57,80 +56,12 @@ func newDiscoveryService(pipeline queue.Client, kind schema.Kind, metadataRepo m
 		pipeline:     pipeline,
 		kind:         kind,
 		metadataRepo: metadataRepo,
+		nodeRegistry: nodeRegistry,
 	}
 }
 
-func (ds *discoveryService) initialize(ctx context.Context) error {
-	ctxLocal, cancel := context.WithTimeout(ctx, 5*time.Second)
-	groups, err := ds.metadataRepo.GroupRegistry().ListGroup(ctxLocal)
-	cancel()
-	if err != nil {
-		return err
-	}
-	for _, g := range groups {
-		switch ds.kind {
-		case schema.KindMeasure:
-		case schema.KindStream:
-		default:
-			continue
-		}
-		ctxLocal, cancel := context.WithTimeout(ctx, 5*time.Second)
-		shards, innerErr := ds.metadataRepo.ShardRegistry().ListShard(ctxLocal, schema.ListOpt{Group: g.Metadata.Name})
-		cancel()
-		if innerErr != nil {
-			return innerErr
-		}
-		for _, s := range shards {
-			ds.shardRepo.OnAddOrUpdate(schema.Metadata{
-				TypeMeta: schema.TypeMeta{
-					Kind:  schema.KindShard,
-					Name:  s.Metadata.Name,
-					Group: s.Metadata.Group,
-				},
-				Spec: s,
-			})
-		}
-
-		switch ds.kind {
-		case schema.KindMeasure:
-			ctxLocal, cancel = context.WithTimeout(ctx, 5*time.Second)
-			mm, innerErr := ds.metadataRepo.MeasureRegistry().ListMeasure(ctxLocal, schema.ListOpt{Group: g.Metadata.Name})
-			cancel()
-			if innerErr != nil {
-				return innerErr
-			}
-			for _, m := range mm {
-				ds.entityRepo.OnAddOrUpdate(schema.Metadata{
-					TypeMeta: schema.TypeMeta{
-						Kind:  schema.KindMeasure,
-						Name:  m.Metadata.Name,
-						Group: m.Metadata.Group,
-					},
-					Spec: m,
-				})
-			}
-		case schema.KindStream:
-			ctxLocal, cancel = context.WithTimeout(ctx, 5*time.Second)
-			ss, innerErr := ds.metadataRepo.StreamRegistry().ListStream(ctxLocal, schema.ListOpt{Group: g.Metadata.Name})
-			cancel()
-			if innerErr != nil {
-				return innerErr
-			}
-			for _, s := range ss {
-				ds.entityRepo.OnAddOrUpdate(schema.Metadata{
-					TypeMeta: schema.TypeMeta{
-						Kind:  schema.KindStream,
-						Name:  s.Metadata.Name,
-						Group: s.Metadata.Group,
-					},
-					Spec: s,
-				})
-			}
-		default:
-			return fmt.Errorf("unsupported kind: %d", ds.kind)
-		}
-	}
-	ds.metadataRepo.RegisterHandler("liaison", schema.KindShard, ds.shardRepo)
+func (ds *discoveryService) initialize() error {
+	ds.metadataRepo.RegisterHandler("liaison", schema.KindGroup, ds.shardRepo)
 	ds.metadataRepo.RegisterHandler("liaison", ds.kind, ds.entityRepo)
 	return nil
 }
@@ -172,28 +103,32 @@ type shardRepo struct {
 	sync.RWMutex
 }
 
-// OnAddOrUpdate implements schema.EventHandler.
 func (s *shardRepo) OnAddOrUpdate(schemaMetadata schema.Metadata) {
-	if schemaMetadata.Kind != schema.KindShard {
+	if schemaMetadata.Kind != schema.KindGroup {
 		return
 	}
-	shard := schemaMetadata.Spec.(*databasev1.Shard)
-	idx := getID(shard.GetMetadata())
+	group := schemaMetadata.Spec.(*commonv1.Group)
+	if group.ResourceOpts == nil || group.Catalog == commonv1.Catalog_CATALOG_UNSPECIFIED {
+		return
+	}
+	idx := getID(group.GetMetadata())
 	if le := s.log.Debug(); le.Enabled() {
-		le.Stringer("id", idx).Uint32("total", shard.Total).Msg("shard added or updated")
+		le.Stringer("id", idx).Uint32("total", group.ResourceOpts.ShardNum).Msg("shard added or updated")
 	}
 	s.RWMutex.Lock()
 	defer s.RWMutex.Unlock()
-	s.shardEventsMap[idx] = shard.Total
+	s.shardEventsMap[idx] = group.ResourceOpts.ShardNum
 }
 
-// OnDelete implements schema.EventHandler.
 func (s *shardRepo) OnDelete(schemaMetadata schema.Metadata) {
-	if schemaMetadata.Kind != schema.KindShard {
+	if schemaMetadata.Kind != schema.KindGroup {
+		return
+	}
+	group := schemaMetadata.Spec.(*commonv1.Group)
+	if group.ResourceOpts == nil || group.Catalog == commonv1.Catalog_CATALOG_UNSPECIFIED {
 		return
 	}
-	shard := schemaMetadata.Spec.(*databasev1.Shard)
-	idx := getID(shard.GetMetadata())
+	idx := getID(group.GetMetadata())
 	if le := s.log.Debug(); le.Enabled() {
 		le.Stringer("id", idx).Msg("shard deleted")
 	}
diff --git a/banyand/liaison/grpc/measure.go b/banyand/liaison/grpc/measure.go
index 90cf7632..4b3f7da9 100644
--- a/banyand/liaison/grpc/measure.go
+++ b/banyand/liaison/grpc/measure.go
@@ -113,11 +113,18 @@ func (ms *measureService) Write(measure measurev1.MeasureService_WriteServer) er
 			SeriesHash:   tsdb.HashEntity(entity),
 			EntityValues: tagValues.Encode(),
 		}
-		// TODO: set node id
-		message := bus.NewMessageWithNode(bus.MessageID(time.Now().UnixNano()), "todo", iwr)
+		nodeID, errPickNode := ms.nodeRegistry.Locate(writeRequest.GetMetadata().GetGroup(), writeRequest.GetMetadata().GetName(), uint32(shardID))
+		if errPickNode != nil {
+			ms.sampled.Error().Err(errPickNode).RawJSON("written", logger.Proto(writeRequest)).Msg("failed to pick an available node")
+			reply(writeRequest.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeRequest.GetMessageId(), measure, ms.sampled)
+			continue
+		}
+		message := bus.NewMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
 		_, errWritePub := publisher.Publish(data.TopicMeasureWrite, message)
 		if errWritePub != nil {
-			ms.sampled.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeRequest)).Msg("failed to send a message")
+			ms.sampled.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeRequest)).Str("nodeID", nodeID).Msg("failed to send a message")
+			reply(writeRequest.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeRequest.GetMessageId(), measure, ms.sampled)
+			continue
 		}
 		reply(nil, modelv1.Status_STATUS_SUCCEED, writeRequest.GetMessageId(), measure, ms.sampled)
 	}
diff --git a/banyand/liaison/grpc/node.go b/banyand/liaison/grpc/node.go
new file mode 100644
index 00000000..73529604
--- /dev/null
+++ b/banyand/liaison/grpc/node.go
@@ -0,0 +1,109 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package grpc
+
+import (
+	"sort"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
+	"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
+	"github.com/apache/skywalking-banyandb/banyand/queue"
+)
+
+var (
+	_ schema.EventHandler = (*clusterNodeService)(nil)
+	_ NodeRegistry        = (*clusterNodeService)(nil)
+)
+
+// NodeRegistry is for locating data node with group/name of the metadata
+// together with the shardID calculated from the incoming data.
+type NodeRegistry interface {
+	Locate(group, name string, shardID uint32) (string, error)
+}
+
+type clusterNodeService struct {
+	metaRepo  queue.Client
+	nodes     []string
+	nodeMutex sync.RWMutex
+	sync.Once
+}
+
+// NewClusterNodeRegistry creates a cluster node registry.
+func NewClusterNodeRegistry(metaRepo queue.Client) NodeRegistry {
+	nr := &clusterNodeService{
+		metaRepo: metaRepo,
+	}
+	metaRepo.Register(nr)
+	return nr
+}
+
+func (n *clusterNodeService) Locate(_, _ string, shardID uint32) (string, error) {
+	// Use round-robin to select the node.
+	n.nodeMutex.RLock()
+	defer n.nodeMutex.RUnlock()
+	if len(n.nodes) == 0 {
+		return "", errors.New("no node available")
+	}
+	return n.nodes[shardID%uint32(len(n.nodes))], nil
+}
+
+func (n *clusterNodeService) OnAddOrUpdate(metadata schema.Metadata) {
+	switch metadata.Kind {
+	case schema.KindNode:
+		n.nodeMutex.Lock()
+		defer n.nodeMutex.Unlock()
+		for _, node := range n.nodes {
+			if node == metadata.Spec.(*databasev1.Node).Metadata.Name {
+				return
+			}
+		}
+		n.nodes = append(n.nodes, metadata.Spec.(*databasev1.Node).Metadata.Name)
+		sort.Strings(n.nodes)
+	default:
+	}
+}
+
+func (n *clusterNodeService) OnDelete(metadata schema.Metadata) {
+	switch metadata.Kind {
+	case schema.KindNode:
+		n.nodeMutex.Lock()
+		defer n.nodeMutex.Unlock()
+		for i, node := range n.nodes {
+			if node == metadata.Spec.(*databasev1.Node).Metadata.Name {
+				n.nodes = append(n.nodes[:i], n.nodes[i+1:]...)
+				break
+			}
+		}
+	default:
+	}
+}
+
+type localNodeService struct{}
+
+// NewLocalNodeRegistry creates a local(fake) node registry.
+func NewLocalNodeRegistry() NodeRegistry {
+	return localNodeService{}
+}
+
+// Locate of localNodeService always returns local.
+func (localNodeService) Locate(_, _ string, _ uint32) (string, error) {
+	return "local", nil
+}
diff --git a/banyand/liaison/grpc/registry_test.go b/banyand/liaison/grpc/registry_test.go
index 6c86d780..6dff21fa 100644
--- a/banyand/liaison/grpc/registry_test.go
+++ b/banyand/liaison/grpc/registry_test.go
@@ -178,7 +178,7 @@ func setupForRegistry() func() {
 	metaSvc, err := metadata.NewService(context.TODO())
 	Expect(err).NotTo(HaveOccurred())
 
-	tcp := grpc.NewServer(context.TODO(), pipeline, metaSvc)
+	tcp := grpc.NewServer(context.TODO(), pipeline, metaSvc, grpc.NewLocalNodeRegistry())
 	preloadStreamSvc := &preloadStreamService{metaSvc: metaSvc}
 	var flags []string
 	metaPath, metaDeferFunc, err := test.NewSpace()
diff --git a/banyand/liaison/grpc/server.go b/banyand/liaison/grpc/server.go
index cc7e0817..a89b7004 100644
--- a/banyand/liaison/grpc/server.go
+++ b/banyand/liaison/grpc/server.go
@@ -91,12 +91,12 @@ type server struct {
 }
 
 // NewServer returns a new gRPC server.
-func NewServer(_ context.Context, pipeline queue.Client, schemaRegistry metadata.Repo) Server {
+func NewServer(_ context.Context, pipeline queue.Client, schemaRegistry metadata.Repo, nodeRegistry NodeRegistry) Server {
 	streamSVC := &streamService{
-		discoveryService: newDiscoveryService(pipeline, schema.KindStream, schemaRegistry),
+		discoveryService: newDiscoveryService(pipeline, schema.KindStream, schemaRegistry, nodeRegistry),
 	}
 	measureSVC := &measureService{
-		discoveryService: newDiscoveryService(pipeline, schema.KindMeasure, schemaRegistry),
+		discoveryService: newDiscoveryService(pipeline, schema.KindMeasure, schemaRegistry, nodeRegistry),
 	}
 	s := &server{
 		pipeline:   pipeline,
@@ -128,7 +128,7 @@ func NewServer(_ context.Context, pipeline queue.Client, schemaRegistry metadata
 	return s
 }
 
-func (s *server) PreRun(ctx context.Context) error {
+func (s *server) PreRun(_ context.Context) error {
 	s.log = logger.GetLogger("liaison-grpc")
 	s.streamSVC.setLogger(s.log)
 	s.measureSVC.setLogger(s.log)
@@ -138,7 +138,7 @@ func (s *server) PreRun(ctx context.Context) error {
 	}
 	for _, c := range components {
 		c.SetLogger(s.log)
-		if err := c.initialize(ctx); err != nil {
+		if err := c.initialize(); err != nil {
 			return err
 		}
 	}
diff --git a/banyand/liaison/grpc/stream.go b/banyand/liaison/grpc/stream.go
index 0fbee9a4..0ae2b30d 100644
--- a/banyand/liaison/grpc/stream.go
+++ b/banyand/liaison/grpc/stream.go
@@ -115,10 +115,17 @@ func (s *streamService) Write(stream streamv1.StreamService_WriteServer) error {
 		if s.log.Debug().Enabled() {
 			iwr.EntityValues = tagValues.Encode()
 		}
-		message := bus.NewMessageWithNode(bus.MessageID(time.Now().UnixNano()), "TODO", iwr)
+		nodeID, errPickNode := s.nodeRegistry.Locate(writeEntity.GetMetadata().GetGroup(), writeEntity.GetMetadata().GetName(), uint32(shardID))
+		if errPickNode != nil {
+			s.sampled.Error().Err(errPickNode).RawJSON("written", logger.Proto(writeEntity)).Msg("failed to pick an available node")
+			reply(writeEntity.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeEntity.GetMessageId(), stream, s.sampled)
+			continue
+		}
+		message := bus.NewMessageWithNode(bus.MessageID(time.Now().UnixNano()), nodeID, iwr)
 		_, errWritePub := publisher.Publish(data.TopicStreamWrite, message)
 		if errWritePub != nil {
-			s.sampled.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeEntity)).Msg("failed to send a message")
+			s.sampled.Error().Err(errWritePub).RawJSON("written", logger.Proto(writeEntity)).Str("nodeID", nodeID).Msg("failed to send a message")
+			reply(writeEntity.GetMetadata(), modelv1.Status_STATUS_INTERNAL_ERROR, writeEntity.GetMessageId(), stream, s.sampled)
 		}
 		reply(nil, modelv1.Status_STATUS_SUCCEED, writeEntity.GetMessageId(), stream, s.sampled)
 	}
diff --git a/banyand/metadata/allocator.go b/banyand/metadata/allocator.go
deleted file mode 100644
index b85470d3..00000000
--- a/banyand/metadata/allocator.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package metadata
-
-import (
-	"context"
-	"time"
-
-	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
-	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
-	"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
-	"github.com/apache/skywalking-banyandb/pkg/logger"
-)
-
-var _ schema.EventHandler = (*allocator)(nil)
-
-type allocator struct {
-	schemaRegistry schema.Registry
-	l              *logger.Logger
-}
-
-func newAllocator(schemaRegistry schema.Registry, logger *logger.Logger) *allocator {
-	return &allocator{
-		schemaRegistry: schemaRegistry,
-		l:              logger,
-	}
-}
-
-// OnAddOrUpdate implements EventHandler.
-func (a *allocator) OnAddOrUpdate(metadata schema.Metadata) {
-	switch metadata.Kind {
-	case schema.KindGroup:
-		groupSchema := metadata.Spec.(*commonv1.Group)
-		if groupSchema.Catalog == commonv1.Catalog_CATALOG_UNSPECIFIED {
-			return
-		}
-		shardNum := groupSchema.GetResourceOpts().GetShardNum()
-		syncShard := func(id uint64) error {
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel()
-			return a.schemaRegistry.CreateOrUpdateShard(ctx, &databasev1.Shard{
-				Id:    id,
-				Total: shardNum,
-				Metadata: &commonv1.Metadata{
-					Name: groupSchema.GetMetadata().GetName(),
-				},
-				Node: "TODO",
-			})
-		}
-		for i := 0; i < int(shardNum); i++ {
-			_ = syncShard(uint64(i))
-		}
-	case schema.KindNode:
-		// TODO: handle node
-	default:
-		return
-	}
-}
-
-// OnDelete implements EventHandler.
-func (*allocator) OnDelete(schema.Metadata) {
-	// TODO: handle delete
-}
diff --git a/banyand/metadata/client.go b/banyand/metadata/client.go
index e584a123..9a166d38 100644
--- a/banyand/metadata/client.go
+++ b/banyand/metadata/client.go
@@ -29,7 +29,6 @@ import (
 	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
 	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
 	"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
-	"github.com/apache/skywalking-banyandb/pkg/logger"
 	"github.com/apache/skywalking-banyandb/pkg/run"
 )
 
@@ -47,7 +46,6 @@ func NewClient(_ context.Context) (Service, error) {
 type clientService struct {
 	namespace      string
 	schemaRegistry schema.Registry
-	alc            *allocator
 	closer         *run.Closer
 	endpoints      []string
 }
@@ -91,7 +89,7 @@ func (s *clientService) PreRun(ctx context.Context) error {
 	nodeRoles := val.([]databasev1.Role)
 	ctxRegister, cancel := context.WithTimeout(ctx, time.Second*5)
 	defer cancel()
-	if err = s.schemaRegistry.RegisterNode(ctxRegister, &databasev1.Node{
+	return s.schemaRegistry.RegisterNode(ctxRegister, &databasev1.Node{
 		Metadata: &commonv1.Metadata{
 			Name: node.NodeID,
 		},
@@ -99,12 +97,7 @@ func (s *clientService) PreRun(ctx context.Context) error {
 		HttpAddress: node.HTTPAddress,
 		Roles:       nodeRoles,
 		CreatedAt:   timestamppb.Now(),
-	}); err != nil {
-		return err
-	}
-	s.alc = newAllocator(s.schemaRegistry, logger.GetLogger(s.Name()).Named("allocator"))
-	s.schemaRegistry.RegisterHandler("shard-allocator", schema.KindGroup|schema.KindNode, s.alc)
-	return nil
+	})
 }
 
 func (s *clientService) Serve() run.StopNotify {
@@ -149,10 +142,6 @@ func (s *clientService) PropertyRegistry() schema.Property {
 	return s.schemaRegistry
 }
 
-func (s *clientService) ShardRegistry() schema.Shard {
-	return s.schemaRegistry
-}
-
 func (s *clientService) Name() string {
 	return "metadata"
 }
diff --git a/banyand/metadata/metadata.go b/banyand/metadata/metadata.go
index 58d10318..a5c76de5 100644
--- a/banyand/metadata/metadata.go
+++ b/banyand/metadata/metadata.go
@@ -46,7 +46,6 @@ type Repo interface {
 	GroupRegistry() schema.Group
 	TopNAggregationRegistry() schema.TopNAggregation
 	PropertyRegistry() schema.Property
-	ShardRegistry() schema.Shard
 	RegisterHandler(string, schema.Kind, schema.EventHandler)
 }
 
diff --git a/banyand/metadata/schema/checker.go b/banyand/metadata/schema/checker.go
index cdedc54b..4a336dc4 100644
--- a/banyand/metadata/schema/checker.go
+++ b/banyand/metadata/schema/checker.go
@@ -88,13 +88,6 @@ var checkerMap = map[Kind]equalityChecker{
 			protocmp.IgnoreFields(&databasev1.Node{}, "created_at"),
 			protocmp.Transform())
 	},
-	KindShard: func(a, b proto.Message) bool {
-		return cmp.Equal(a, b,
-			protocmp.IgnoreUnknown(),
-			protocmp.IgnoreFields(&databasev1.Shard{}, "updated_at"),
-			protocmp.IgnoreFields(&commonv1.Metadata{}, "id", "create_revision", "mod_revision"),
-			protocmp.Transform())
-	},
 	KindMask: func(a, b proto.Message) bool {
 		return false
 	},
diff --git a/banyand/metadata/schema/kind.go b/banyand/metadata/schema/kind.go
index 4f406f86..1a0573bb 100644
--- a/banyand/metadata/schema/kind.go
+++ b/banyand/metadata/schema/kind.go
@@ -42,10 +42,9 @@ const (
 	KindTopNAggregation
 	KindProperty
 	KindNode
-	KindShard
 	KindMask = KindGroup | KindStream | KindMeasure |
 		KindIndexRuleBinding | KindIndexRule |
-		KindTopNAggregation | KindProperty | KindNode | KindShard
+		KindTopNAggregation | KindProperty | KindNode
 	KindSize = 9
 )
 
@@ -67,8 +66,6 @@ func (k Kind) key() string {
 		return propertyKeyPrefix
 	case KindNode:
 		return nodeKeyPrefix
-	case KindShard:
-		return shardKeyPrefix
 	default:
 		return "unknown"
 	}
@@ -97,8 +94,6 @@ func (k Kind) Unmarshal(kv *mvccpb.KeyValue) (Metadata, error) {
 		m = &databasev1.TopNAggregation{}
 	case KindNode:
 		m = &databasev1.Node{}
-	case KindShard:
-		m = &databasev1.Shard{}
 	default:
 		return Metadata{}, errUnsupportedEntityType
 	}
@@ -143,8 +138,6 @@ func (k Kind) String() string {
 		return "property"
 	case KindNode:
 		return "node"
-	case KindShard:
-		return "shard"
 	default:
 		return "unknown"
 	}
diff --git a/banyand/metadata/schema/schema.go b/banyand/metadata/schema/schema.go
index 0c7095af..dd4570ca 100644
--- a/banyand/metadata/schema/schema.go
+++ b/banyand/metadata/schema/schema.go
@@ -54,7 +54,6 @@ type Registry interface {
 	TopNAggregation
 	Property
 	Node
-	Shard
 	RegisterHandler(string, Kind, EventHandler)
 }
 
@@ -111,11 +110,6 @@ func (m Metadata) key() (string, error) {
 		}), nil
 	case KindNode:
 		return formatNodeKey(m.Name), nil
-	case KindShard:
-		return formatShardKey(&commonv1.Metadata{
-			Group: m.Group,
-			Name:  m.Name,
-		}), nil
 	default:
 		return "", errUnsupportedEntityType
 	}
@@ -203,9 +197,3 @@ type Node interface {
 	ListNode(ctx context.Context, role databasev1.Role) ([]*databasev1.Node, error)
 	RegisterNode(ctx context.Context, node *databasev1.Node) error
 }
-
-// Shard allows CRUD shard schemas in a group.
-type Shard interface {
-	CreateOrUpdateShard(ctx context.Context, shard *databasev1.Shard) error
-	ListShard(ctx context.Context, opt ListOpt) ([]*databasev1.Shard, error)
-}
diff --git a/banyand/metadata/schema/shard.go b/banyand/metadata/schema/shard.go
deleted file mode 100644
index c6499931..00000000
--- a/banyand/metadata/schema/shard.go
+++ /dev/null
@@ -1,74 +0,0 @@
-// Licensed to Apache Software Foundation (ASF) under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Apache Software Foundation (ASF) licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package schema
-
-import (
-	"context"
-
-	"github.com/pkg/errors"
-	"google.golang.org/protobuf/types/known/timestamppb"
-
-	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
-	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
-)
-
-var shardKeyPrefix = "/shards/"
-
-func (e *etcdSchemaRegistry) CreateOrUpdateShard(ctx context.Context, shard *databasev1.Shard) error {
-	if shard.UpdatedAt != nil {
-		shard.UpdatedAt = timestamppb.Now()
-	}
-	md := Metadata{
-		TypeMeta: TypeMeta{
-			Kind:  KindShard,
-			Group: shard.GetMetadata().GetGroup(),
-			Name:  shard.GetMetadata().GetName(),
-		},
-		Spec: shard,
-	}
-	_, err := e.update(ctx, md)
-	if err == nil {
-		return nil
-	}
-	if errors.Is(err, ErrGRPCResourceNotFound) {
-		shard.CreatedAt = shard.UpdatedAt
-		md.Spec = shard
-		_, err = e.create(ctx, md)
-		return err
-	}
-	return err
-}
-
-func (e *etcdSchemaRegistry) ListShard(ctx context.Context, opt ListOpt) ([]*databasev1.Shard, error) {
-	if opt.Group == "" {
-		return nil, BadRequest("group", "group should not be empty")
-	}
-	messages, err := e.listWithPrefix(ctx, listPrefixesForEntity(opt.Group, shardKeyPrefix), KindShard)
-	if err != nil {
-		return nil, err
-	}
-	entities := make([]*databasev1.Shard, 0, len(messages))
-	for _, message := range messages {
-		entities = append(entities, message.(*databasev1.Shard))
-	}
-	return entities, nil
-}
-
-func formatShardKey(metadata *commonv1.Metadata) string {
-	return formatKey(shardKeyPrefix, metadata)
-}
diff --git a/banyand/queue/local.go b/banyand/queue/local.go
index c5b8dc42..a6872881 100644
--- a/banyand/queue/local.go
+++ b/banyand/queue/local.go
@@ -19,6 +19,7 @@
 package queue
 
 import (
+	"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
 	"github.com/apache/skywalking-banyandb/pkg/bus"
 	"github.com/apache/skywalking-banyandb/pkg/run"
 )
@@ -84,6 +85,9 @@ func (*local) GetPort() *uint32 {
 	return nil
 }
 
+func (*local) Register(schema.EventHandler) {
+}
+
 type localBatchPublisher struct {
 	local *bus.Bus
 }
diff --git a/banyand/queue/pub/client.go b/banyand/queue/pub/client.go
index c5c68139..aa9dbbfe 100644
--- a/banyand/queue/pub/client.go
+++ b/banyand/queue/pub/client.go
@@ -88,6 +88,9 @@ func (p *pub) OnAddOrUpdate(md schema.Metadata) {
 	}
 	c := clusterv1.NewServiceClient(conn)
 	p.clients[name] = &client{conn: conn, client: c}
+	if p.handler != nil {
+		p.handler.OnAddOrUpdate(md)
+	}
 }
 
 func (p *pub) OnDelete(md schema.Metadata) {
@@ -112,5 +115,8 @@ func (p *pub) OnDelete(md schema.Metadata) {
 			client.conn.Close() // Close the client connection
 		}
 		delete(p.clients, name)
+		if p.handler != nil {
+			p.handler.OnDelete(md)
+		}
 	}
 }
diff --git a/banyand/queue/pub/pub.go b/banyand/queue/pub/pub.go
index b72e9a07..9e00b13a 100644
--- a/banyand/queue/pub/pub.go
+++ b/banyand/queue/pub/pub.go
@@ -46,13 +46,17 @@ var (
 
 type pub struct {
 	metadata metadata.Repo
+	handler  schema.EventHandler
 	log      *logger.Logger
 	clients  map[string]*client
 	closer   *run.Closer
 	mu       sync.RWMutex
 }
 
-// GracefulStop implements run.Service.
+func (p *pub) Register(handler schema.EventHandler) {
+	p.handler = handler
+}
+
 func (p *pub) GracefulStop() {
 	p.closer.Done()
 	p.closer.CloseThenWait()
@@ -133,12 +137,10 @@ func New(metadata metadata.Repo) queue.Client {
 	}
 }
 
-// Name implements run.PreRunner.
 func (*pub) Name() string {
 	return "queue-client"
 }
 
-// PreRun implements run.PreRunner.
 func (p *pub) PreRun(context.Context) error {
 	p.log = logger.GetLogger("server-queue")
 	p.metadata.RegisterHandler("queue-client", schema.KindNode, p)
diff --git a/banyand/queue/queue.go b/banyand/queue/queue.go
index 7bb8c531..d518d331 100644
--- a/banyand/queue/queue.go
+++ b/banyand/queue/queue.go
@@ -20,6 +20,7 @@ package queue
 import (
 	"io"
 
+	"github.com/apache/skywalking-banyandb/banyand/metadata/schema"
 	"github.com/apache/skywalking-banyandb/pkg/bus"
 	"github.com/apache/skywalking-banyandb/pkg/run"
 )
@@ -39,6 +40,7 @@ type Client interface {
 	bus.Publisher
 	bus.Broadcaster
 	NewBatchPublisher() BatchPublisher
+	Register(schema.EventHandler)
 }
 
 // Server is the interface for receiving data from the queue.
diff --git a/pkg/cmdsetup/liaison.go b/pkg/cmdsetup/liaison.go
index 3ef2eb44..e1178ef3 100644
--- a/pkg/cmdsetup/liaison.go
+++ b/pkg/cmdsetup/liaison.go
@@ -43,7 +43,7 @@ func newLiaisonCmd(runners ...run.Unit) *cobra.Command {
 		l.Fatal().Err(err).Msg("failed to initiate metadata service")
 	}
 	pipeline := pub.New(metaSvc)
-	grpcServer := grpc.NewServer(ctx, pipeline, metaSvc)
+	grpcServer := grpc.NewServer(ctx, pipeline, metaSvc, grpc.NewClusterNodeRegistry(pipeline))
 	profSvc := observability.NewProfService()
 	metricSvc := observability.NewMetricService()
 	httpServer := http.NewServer()
diff --git a/pkg/cmdsetup/standalone.go b/pkg/cmdsetup/standalone.go
index f0dca2f2..3c659b09 100644
--- a/pkg/cmdsetup/standalone.go
+++ b/pkg/cmdsetup/standalone.go
@@ -57,7 +57,7 @@ func newStandaloneCmd(runners ...run.Unit) *cobra.Command {
 	if err != nil {
 		l.Fatal().Err(err).Msg("failed to initiate query processor")
 	}
-	grpcServer := grpc.NewServer(ctx, pipeline, metaSvc)
+	grpcServer := grpc.NewServer(ctx, pipeline, metaSvc, grpc.NewLocalNodeRegistry())
 	profSvc := observability.NewProfService()
 	metricSvc := observability.NewMetricService()
 	httpServer := http.NewServer()