You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dubbo.apache.org by xu...@apache.org on 2026/05/06 03:06:14 UTC

(dubbo-go-pixiu) branch develop updated: fix(cluster): tighten runtime consistency (#919)

This is an automated email from the ASF dual-hosted git repository.

Alanxtl pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/dubbo-go-pixiu.git


The following commit(s) were added to refs/heads/develop by this push:
     new 55c62109 fix(cluster): tighten runtime consistency (#919)
55c62109 is described below

commit 55c62109a5f73633ce4755cbd0a8c92f4980c5fd
Author: 承潜 <29...@qq.com>
AuthorDate: Wed May 6 11:06:09 2026 +0800

    fix(cluster): tighten runtime consistency (#919)
    
    * fix(cluster): tighten runtime consistency
    
    * fix(cluster): address runtime consistency review comments
---
 pkg/cluster/healthcheck/healthcheck.go |  25 ++-
 pkg/server/cluster_manager.go          | 217 ++++++++++++++++++-----
 pkg/server/cluster_manager_test.go     | 305 +++++++++++++++++++++++++++++++++
 3 files changed, 496 insertions(+), 51 deletions(-)

diff --git a/pkg/cluster/healthcheck/healthcheck.go b/pkg/cluster/healthcheck/healthcheck.go
index 5a0d0921..d899bd35 100644
--- a/pkg/cluster/healthcheck/healthcheck.go
+++ b/pkg/cluster/healthcheck/healthcheck.go
@@ -212,13 +212,7 @@ func (hc *HealthChecker) getCheckInterval() time.Duration {
 }
 
 func (c *EndpointChecker) Start() {
-	defer func() {
-		if r := recover(); r != nil {
-			logger.Warnf("[health check] node checker panic %v\n%s", r, string(debug.Stack()))
-		}
-		c.checkTimer.Stop()
-		c.checkTimeout.Stop()
-	}()
+	defer c.cleanupStart()
 	c.checkTimer = gxtime.AfterFunc(c.HealthChecker.initialDelay, c.OnCheck)
 	for {
 		select {
@@ -256,6 +250,23 @@ func (c *EndpointChecker) Start() {
 	}
 }
 
+func (c *EndpointChecker) cleanupStart() {
+	if r := recover(); r != nil {
+		logger.Warnf("[health check] node checker panic %v\n%s", r, string(debug.Stack()))
+	}
+	c.stopCheckTimers()
+}
+
+func (c *EndpointChecker) stopCheckTimers() {
+	// Early Stop may run before timers are assigned.
+	if c.checkTimer != nil {
+		c.checkTimer.Stop()
+	}
+	if c.checkTimeout != nil {
+		c.checkTimeout.Stop()
+	}
+}
+
 func (c *EndpointChecker) Stop() {
 	c.once.Do(func() {
 		close(c.stop)
diff --git a/pkg/server/cluster_manager.go b/pkg/server/cluster_manager.go
index 09b60ba2..8b44ba65 100644
--- a/pkg/server/cluster_manager.go
+++ b/pkg/server/cluster_manager.go
@@ -143,17 +143,35 @@ func (cm *ClusterManager) NewStore(version int32) *ClusterStore {
 	return &ClusterStore{Version: version, clustersMap: map[string]*cluster.Cluster{}}
 }
 
+// CompareAndSetStore swaps the store only when versions match.
+// Version mismatch must leave both stores and runtime clusters untouched.
 func (cm *ClusterManager) CompareAndSetStore(store *ClusterStore) bool {
+	swapped, replacedClusters := cm.compareAndSetStore(store)
+	if !swapped {
+		return false
+	}
+
+	// Stop old runtime after publishing the swap; Stop may touch timers/goroutines.
+	stopClusters(replacedClusters)
+	return true
+}
+
+func (cm *ClusterManager) compareAndSetStore(store *ClusterStore) (bool, []*cluster.Cluster) {
 	cm.rw.Lock()
 	defer cm.rw.Unlock()
 
 	if store.Version != cm.store.Version {
-		return false
+		return false, nil
 	}
 
-	store.carryOverRuntimeStateFrom(cm.store)
+	currentStore := cm.store
+	replacedClusters := store.ensureRuntimeClusters()
+	store.carryOverRuntimeStateFrom(currentStore)
 	cm.store = store
-	return true
+	if store != currentStore {
+		replacedClusters = append(replacedClusters, currentStore.runtimeClustersNotIn(store)...)
+	}
+	return true, replacedClusters
 }
 
 // PickEndpoint picks an endpoint from the cluster by its name and load balancing policy.
@@ -249,7 +267,7 @@ func (cm *ClusterManager) RemoveCluster(namesToDel []string) {
 		for _, name := range namesToDel { // suppose resource to remove and clusters is few
 			if name == c.Name {
 				removed := cm.store.Config[i]
-				cm.store.clustersMap[removed.Name].Stop()
+				stopClusters([]*cluster.Cluster{cm.store.clustersMap[removed.Name]})
 				cm.store.Config[i] = nil
 				delete(cm.store.clustersMap, removed.Name)
 			}
@@ -278,10 +296,15 @@ func (s *ClusterStore) AddCluster(c *model.ClusterConfig) {
 		atomic.AddInt32(&clusterIndex, 1)
 	}
 
-	s.assembleClusterEndpoints(c)
+	s.prepareClusterConfig(c)
 
 	s.Config = append(s.Config, c)
-	s.clustersMap[c.Name] = cluster.NewCluster(c)
+	stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(c.Name, c)})
+}
+
+// prepareClusterConfig rebuilds endpoint defaults and hash from current endpoints.
+func (s *ClusterStore) prepareClusterConfig(c *model.ClusterConfig) {
+	s.assembleClusterEndpoints(c)
 	c.CreateConsistentHash()
 }
 
@@ -309,10 +332,108 @@ func (s *ClusterStore) assembleClusterEndpoints(c *model.ClusterConfig) {
 	}
 }
 
+// replaceClusterRuntime returns the old runtime so callers decide when to stop it.
+func (s *ClusterStore) replaceClusterRuntime(name string, config *model.ClusterConfig) *cluster.Cluster {
+	if s.clustersMap == nil {
+		s.clustersMap = map[string]*cluster.Cluster{}
+	}
+
+	oldRuntime := s.clustersMap[name]
+	s.clustersMap[name] = cluster.NewCluster(config)
+	return oldRuntime
+}
+
+// ensureRuntimeClusters repairs clustersMap to match Config by name and pointer.
+func (s *ClusterStore) ensureRuntimeClusters() []*cluster.Cluster {
+	if s == nil {
+		return nil
+	}
+	if s.clustersMap == nil {
+		s.clustersMap = map[string]*cluster.Cluster{}
+	}
+
+	replacedClusters := make([]*cluster.Cluster, 0)
+	configsByName := make(map[string]*model.ClusterConfig, len(s.Config))
+	for _, clusterConfig := range s.Config {
+		if clusterConfig == nil {
+			continue
+		}
+		s.prepareClusterConfig(clusterConfig)
+		configsByName[clusterConfig.Name] = clusterConfig
+
+		runtimeCluster := s.clustersMap[clusterConfig.Name]
+		if runtimeCluster == nil || runtimeCluster.Config != clusterConfig {
+			if oldRuntime := s.replaceClusterRuntime(clusterConfig.Name, clusterConfig); oldRuntime != nil {
+				replacedClusters = append(replacedClusters, oldRuntime)
+			}
+		}
+	}
+
+	for name, runtimeCluster := range s.clustersMap {
+		if _, ok := configsByName[name]; !ok {
+			replacedClusters = append(replacedClusters, runtimeCluster)
+			delete(s.clustersMap, name)
+		}
+	}
+	return replacedClusters
+}
+
+// runtimeClustersNotIn finds old runtime clusters dropped by the next store.
+func (s *ClusterStore) runtimeClustersNotIn(next *ClusterStore) []*cluster.Cluster {
+	if s == nil {
+		return nil
+	}
+
+	nextClusters := make(map[*cluster.Cluster]struct{})
+	if next != nil {
+		for _, runtimeCluster := range next.clustersMap {
+			if runtimeCluster != nil {
+				nextClusters[runtimeCluster] = struct{}{}
+			}
+		}
+	}
+
+	replacedClusters := make([]*cluster.Cluster, 0)
+	for _, runtimeCluster := range s.clustersMap {
+		if runtimeCluster == nil {
+			continue
+		}
+		if _, ok := nextClusters[runtimeCluster]; !ok {
+			replacedClusters = append(replacedClusters, runtimeCluster)
+		}
+	}
+	return replacedClusters
+}
+
+// stopClusters is nil-safe and avoids stopping the same runtime twice.
+func stopClusters(clusters []*cluster.Cluster) {
+	stopped := make(map[*cluster.Cluster]struct{}, len(clusters))
+	for _, runtimeCluster := range clusters {
+		if runtimeCluster == nil {
+			continue
+		}
+		if _, ok := stopped[runtimeCluster]; ok {
+			continue
+		}
+		stopped[runtimeCluster] = struct{}{}
+		runtimeCluster.Stop()
+	}
+}
+
+// UpdateCluster replaces config/runtime together while preserving the RR cursor.
 func (s *ClusterStore) UpdateCluster(new *model.ClusterConfig) {
 	for i, c := range s.Config {
+		if c == nil {
+			continue
+		}
 		if c.Name == new.Name {
+			s.prepareClusterConfig(new)
+			atomic.StoreUint32(
+				&new.PrePickEndpointIndex,
+				atomic.LoadUint32(&c.PrePickEndpointIndex),
+			)
 			s.Config[i] = new
+			stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(new.Name, new)})
 			return
 		}
 	}
@@ -320,59 +441,67 @@ func (s *ClusterStore) UpdateCluster(new *model.ClusterConfig) {
 }
 
 func (s *ClusterStore) SetEndpoint(clusterName string, endpoint *model.Endpoint) {
-	cluster := s.clustersMap[clusterName]
-	if cluster == nil {
+	clusterConfig := s.findClusterConfig(clusterName)
+	if clusterConfig == nil {
 		c := &model.ClusterConfig{Name: clusterName, LbStr: model.LoadBalancerRoundRobin, Endpoints: []*model.Endpoint{}}
 		s.AddCluster(c)
-		cluster = s.clustersMap[clusterName]
+		clusterConfig = c
 	}
 
-	for _, c := range s.Config {
-		if c.Name == clusterName {
-			for _, e := range c.Endpoints {
-				// endpoint update
-				if e.ID == endpoint.ID {
-					cluster.RemoveEndpoint(e)
-					e.Name = endpoint.Name
-					e.Metadata = endpoint.Metadata
-					e.Address = endpoint.Address
-					cluster.AddEndpoint(e)
-					return
-				}
-			}
-			// endpoint create
-			c.Endpoints = append(c.Endpoints, endpoint)
-			cluster.AddEndpoint(endpoint)
-			if c.ConsistentHash.Hash != nil {
-				c.ConsistentHash.Hash.Add(endpoint.GetHost())
-			}
+	runtimeCluster := s.clustersMap[clusterName]
+	if runtimeCluster == nil || runtimeCluster.Config != clusterConfig {
+		stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(clusterName, clusterConfig)})
+		runtimeCluster = s.clustersMap[clusterName]
+	}
+
+	for _, e := range clusterConfig.Endpoints {
+		if e.ID == endpoint.ID {
+			// Remove before mutating address because healthcheck keys by address.
+			runtimeCluster.RemoveEndpoint(e)
+			e.Name = endpoint.Name
+			e.Metadata = endpoint.Metadata
+			e.Address = endpoint.Address
+			s.prepareClusterConfig(clusterConfig)
+			runtimeCluster.AddEndpoint(e)
 			return
 		}
 	}
+	clusterConfig.Endpoints = append(clusterConfig.Endpoints, endpoint)
+	s.prepareClusterConfig(clusterConfig)
+	runtimeCluster.AddEndpoint(endpoint)
 }
 
 func (s *ClusterStore) DeleteEndpoint(clusterName string, endpointID string) {
-	cluster := s.clustersMap[clusterName]
-	if cluster == nil {
+	clusterConfig := s.findClusterConfig(clusterName)
+	if clusterConfig == nil {
+		logger.Warnf("not found cluster %s", clusterName)
 		return
 	}
-	for _, c := range s.Config {
-		if c.Name == clusterName {
-			for i, e := range c.Endpoints {
-				if e.ID == endpointID {
-					cluster.RemoveEndpoint(e)
-					c.Endpoints = append(c.Endpoints[:i], c.Endpoints[i+1:]...)
-					if c.ConsistentHash.Hash != nil {
-						c.ConsistentHash.Hash.Remove(e.GetHost())
-					}
-					return
-				}
-			}
-			logger.Warnf("not found endpoint %s", endpointID)
+
+	runtimeCluster := s.clustersMap[clusterName]
+	if runtimeCluster == nil || runtimeCluster.Config != clusterConfig {
+		stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(clusterName, clusterConfig)})
+		runtimeCluster = s.clustersMap[clusterName]
+	}
+
+	for i, e := range clusterConfig.Endpoints {
+		if e.ID == endpointID {
+			runtimeCluster.RemoveEndpoint(e)
+			clusterConfig.Endpoints = append(clusterConfig.Endpoints[:i], clusterConfig.Endpoints[i+1:]...)
+			s.prepareClusterConfig(clusterConfig)
 			return
 		}
 	}
-	logger.Warnf("not found cluster %s", clusterName)
+	logger.Warnf("not found endpoint %s", endpointID)
+}
+
+func (s *ClusterStore) findClusterConfig(clusterName string) *model.ClusterConfig {
+	for _, c := range s.Config {
+		if c != nil && c.Name == clusterName {
+			return c
+		}
+	}
+	return nil
 }
 
 func (s *ClusterStore) HasCluster(clusterName string) bool {
diff --git a/pkg/server/cluster_manager_test.go b/pkg/server/cluster_manager_test.go
index dd06e25a..6faf6cb9 100644
--- a/pkg/server/cluster_manager_test.go
+++ b/pkg/server/cluster_manager_test.go
@@ -19,6 +19,7 @@ package server
 
 import (
 	"fmt"
+	"reflect"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -29,6 +30,7 @@ import (
 )
 
 import (
+	"github.com/apache/dubbo-go-pixiu/pkg/cluster"
 	_ "github.com/apache/dubbo-go-pixiu/pkg/cluster/loadbalancer/maglev"     // Register Maglev for cluster-manager tests.
 	_ "github.com/apache/dubbo-go-pixiu/pkg/cluster/loadbalancer/rand"       // Register Rand for cluster-manager tests.
 	_ "github.com/apache/dubbo-go-pixiu/pkg/cluster/loadbalancer/ringhash"   // Register RingHash for cluster-manager tests.
@@ -138,6 +140,283 @@ func TestClusterManager_CompareAndSetStorePreservesRoundRobinCursorAcrossRefresh
 	}
 }
 
+func TestClusterManager_UpdateClusterRebuildsRuntimeCluster(t *testing.T) {
+	oldConfig := testCluster("runtime-update", model.LoadBalancerRoundRobin, []*model.Endpoint{
+		testEndpoint("ep-1", "127.0.0.1", 19300),
+	}, testHealthCheck())
+	cm := testClusterManager(oldConfig)
+	defer stopStoreRuntimes(cm.store)
+
+	oldRuntime := cm.store.clustersMap[oldConfig.Name]
+	if !assert.NotNil(t, oldRuntime) {
+		return
+	}
+	assert.Same(t, oldConfig, oldRuntime.Config)
+	assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+	const expectedCursor uint32 = 11
+	atomic.StoreUint32(&oldConfig.PrePickEndpointIndex, expectedCursor)
+
+	newConfig := testCluster(oldConfig.Name, model.LoadBalancerRoundRobin, []*model.Endpoint{
+		testEndpoint("ep-2", "127.0.0.1", 19301),
+	}, testHealthCheck())
+
+	cm.UpdateCluster(newConfig)
+
+	newRuntime := cm.store.clustersMap[oldConfig.Name]
+	if !assert.NotNil(t, newRuntime) {
+		return
+	}
+	assert.NotSame(t, oldRuntime, newRuntime)
+	assert.Same(t, newConfig, newRuntime.Config)
+	assert.Same(t, newConfig, cm.store.Config[0])
+	assert.Equal(t, expectedCursor, atomic.LoadUint32(&newConfig.PrePickEndpointIndex))
+	assert.Equal(t, 0, healthCheckersLen(oldRuntime))
+	assert.Greater(t, healthCheckersLen(newRuntime), 0)
+}
+
+func TestClusterManager_CompareAndSetStoreVersionMismatchHasNoSideEffects(t *testing.T) {
+	currentConfig := testCluster("cas-version-mismatch", model.LoadBalancerRoundRobin, []*model.Endpoint{
+		testEndpoint("ep-1", "127.0.0.1", 19310),
+	}, testHealthCheck())
+	cm := testClusterManager(currentConfig)
+	defer stopStoreRuntimes(cm.store)
+
+	oldStore := cm.store
+	oldRuntime := oldStore.clustersMap[currentConfig.Name]
+	if !assert.NotNil(t, oldRuntime) {
+		return
+	}
+	assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+	candidateConfig := testCluster(currentConfig.Name, model.LoadBalancerRoundRobin, []*model.Endpoint{
+		testEndpoint("ep-2", "127.0.0.1", 19311),
+	})
+	candidate := &ClusterStore{
+		Config:  []*model.ClusterConfig{candidateConfig},
+		Version: oldStore.Version + 1,
+	}
+
+	assert.False(t, cm.CompareAndSetStore(candidate))
+	assert.Same(t, oldStore, cm.store)
+	assert.Nil(t, candidate.clustersMap)
+	assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+}
+
+func TestClusterManager_CompareAndSetStoreEnsuresRuntimeAndStopsOld(t *testing.T) {
+	oldConfig := testCluster("cas-runtime-refresh", model.LoadBalancerRoundRobin, []*model.Endpoint{
+		testEndpoint("ep-1", "127.0.0.1", 19320),
+	}, testHealthCheck())
+	cm := testClusterManager(oldConfig)
+
+	oldRuntime := cm.store.clustersMap[oldConfig.Name]
+	if !assert.NotNil(t, oldRuntime) {
+		return
+	}
+	assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+	const expectedCursor uint32 = 17
+	atomic.StoreUint32(&oldConfig.PrePickEndpointIndex, expectedCursor)
+
+	newConfig := testCluster(oldConfig.Name, model.LoadBalancerRoundRobin, []*model.Endpoint{
+		testEndpoint("ep-2", "127.0.0.1", 19321),
+	}, testHealthCheck())
+	candidate := &ClusterStore{
+		Config:  []*model.ClusterConfig{newConfig},
+		Version: cm.store.Version,
+	}
+
+	assert.True(t, cm.CompareAndSetStore(candidate))
+	defer stopStoreRuntimes(cm.store)
+
+	newRuntime := candidate.clustersMap[newConfig.Name]
+	if !assert.NotNil(t, newRuntime) {
+		return
+	}
+	assert.Same(t, candidate, cm.store)
+	assert.NotSame(t, oldRuntime, newRuntime)
+	assert.Same(t, newConfig, newRuntime.Config)
+	assert.Equal(t, expectedCursor, atomic.LoadUint32(&newConfig.PrePickEndpointIndex))
+	assert.Equal(t, 0, healthCheckersLen(oldRuntime))
+	assert.Greater(t, healthCheckersLen(newRuntime), 0)
+}
+
+func TestClusterManager_CompareAndSetStoreStopsRemovedRuntime(t *testing.T) {
+	oldConfig := testCluster("cas-runtime-removed", model.LoadBalancerRoundRobin, []*model.Endpoint{
+		testEndpoint("ep-1", "127.0.0.1", 19325),
+	}, testHealthCheck())
+	cm := testClusterManager(oldConfig)
+
+	oldRuntime := cm.store.clustersMap[oldConfig.Name]
+	if !assert.NotNil(t, oldRuntime) {
+		return
+	}
+	assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+	candidate := &ClusterStore{
+		Version: cm.store.Version,
+	}
+
+	assert.True(t, cm.CompareAndSetStore(candidate))
+	assert.Same(t, candidate, cm.store)
+	assert.Empty(t, cm.store.Config)
+	assert.NotContains(t, cm.store.clustersMap, oldConfig.Name)
+	assert.Equal(t, 0, healthCheckersLen(oldRuntime))
+}
+
+func TestClusterStore_EnsureRuntimeClustersRepairsRuntimeMap(t *testing.T) {
+	t.Run("nil map", func(t *testing.T) {
+		config := testCluster("ensure-nil-map", model.LoadBalancerRoundRobin, []*model.Endpoint{
+			testEndpoint("ep-1", "127.0.0.1", 19330),
+		})
+		store := &ClusterStore{Config: []*model.ClusterConfig{config}}
+
+		replaced := store.ensureRuntimeClusters()
+		defer stopStoreRuntimes(store)
+
+		assert.Empty(t, replaced)
+		if assert.NotNil(t, store.clustersMap[config.Name]) {
+			assert.Same(t, config, store.clustersMap[config.Name].Config)
+		}
+	})
+
+	t.Run("missing mismatched stale and idempotent", func(t *testing.T) {
+		correctConfig := testCluster("ensure-correct", model.LoadBalancerRoundRobin, []*model.Endpoint{
+			testEndpoint("ep-1", "127.0.0.1", 19331),
+		})
+		missingConfig := testCluster("ensure-missing", model.LoadBalancerRoundRobin, []*model.Endpoint{
+			testEndpoint("ep-2", "127.0.0.1", 19332),
+		})
+		oldMismatchedConfig := testCluster("ensure-mismatch", model.LoadBalancerRoundRobin, []*model.Endpoint{
+			testEndpoint("old", "127.0.0.1", 19333),
+		})
+		newMismatchedConfig := testCluster("ensure-mismatch", model.LoadBalancerRoundRobin, []*model.Endpoint{
+			testEndpoint("new", "127.0.0.1", 19334),
+		})
+		staleConfig := testCluster("ensure-stale", model.LoadBalancerRoundRobin, []*model.Endpoint{
+			testEndpoint("stale", "127.0.0.1", 19335),
+		})
+
+		correctRuntime := cluster.NewCluster(correctConfig)
+		mismatchedRuntime := cluster.NewCluster(oldMismatchedConfig)
+		staleRuntime := cluster.NewCluster(staleConfig)
+		store := &ClusterStore{
+			Config: []*model.ClusterConfig{
+				correctConfig,
+				missingConfig,
+				newMismatchedConfig,
+			},
+			clustersMap: map[string]*cluster.Cluster{
+				correctConfig.Name:       correctRuntime,
+				newMismatchedConfig.Name: mismatchedRuntime,
+				staleConfig.Name:         staleRuntime,
+			},
+		}
+		defer stopStoreRuntimes(store)
+
+		replaced := store.ensureRuntimeClusters()
+		stopClusters(replaced)
+
+		assert.Same(t, correctRuntime, store.clustersMap[correctConfig.Name])
+		if assert.NotNil(t, store.clustersMap[missingConfig.Name]) {
+			assert.Same(t, missingConfig, store.clustersMap[missingConfig.Name].Config)
+		}
+		if assert.NotNil(t, store.clustersMap[newMismatchedConfig.Name]) {
+			assert.NotSame(t, mismatchedRuntime, store.clustersMap[newMismatchedConfig.Name])
+			assert.Same(t, newMismatchedConfig, store.clustersMap[newMismatchedConfig.Name].Config)
+		}
+		assert.NotContains(t, store.clustersMap, staleConfig.Name)
+		assert.Contains(t, replaced, mismatchedRuntime)
+		assert.Contains(t, replaced, staleRuntime)
+
+		runtimesAfterRepair := map[string]*cluster.Cluster{}
+		for name, runtime := range store.clustersMap {
+			runtimesAfterRepair[name] = runtime
+		}
+
+		assert.Empty(t, store.ensureRuntimeClusters())
+		assert.Equal(t, runtimesAfterRepair, store.clustersMap)
+	})
+}
+
+func TestClusterManager_SetEndpointUpdateRebuildsConsistentHash(t *testing.T) {
+	tests := []model.LbPolicyType{
+		model.LoadBalancerRingHashing,
+		model.LoadBalancerMaglevHashing,
+	}
+
+	for _, lb := range tests {
+		t.Run(string(lb), func(t *testing.T) {
+			oldEndpoint := testEndpoint("ep-1", "127.0.0.1", 19340)
+			oldHost := oldEndpoint.GetHost()
+			config := testCluster(fmt.Sprintf("hash-update-%s", lb), lb, []*model.Endpoint{oldEndpoint})
+			cm := testClusterManager(config)
+			defer stopStoreRuntimes(cm.store)
+
+			newEndpoint := testEndpoint("ep-1", "127.0.0.2", 19341)
+			newHost := newEndpoint.GetHost()
+			cm.SetEndpoint(config.Name, newEndpoint)
+
+			hash := cm.store.Config[0].ConsistentHash.Hash
+			if !assert.NotNil(t, hash) {
+				return
+			}
+			if hostList, ok := hash.(interface{ Hosts() []string }); ok {
+				hosts := hostList.Hosts()
+				assert.NotContains(t, hosts, oldHost)
+				assert.Contains(t, hosts, newHost)
+				return
+			}
+			assert.False(t, hash.Remove(oldHost))
+			assert.True(t, hash.Remove(newHost))
+		})
+	}
+}
+
+func TestClusterManager_DeleteEndpointRepairsRuntimeAndConsistentHash(t *testing.T) {
+	deletedEndpoint := testEndpoint("ep-1", "127.0.0.1", 19350)
+	remainingEndpoint := testEndpoint("ep-2", "127.0.0.1", 19351)
+	config := testCluster("delete-runtime-repair", model.LoadBalancerRingHashing, []*model.Endpoint{
+		deletedEndpoint,
+		remainingEndpoint,
+	})
+	cm := testClusterManager(config)
+	defer stopStoreRuntimes(cm.store)
+
+	staleConfig := testCluster(config.Name, model.LoadBalancerRingHashing, []*model.Endpoint{
+		testEndpoint("stale", "127.0.0.1", 19352),
+	})
+	staleRuntime := cluster.NewCluster(staleConfig)
+	cm.store.clustersMap[config.Name] = staleRuntime
+
+	deletedHost := deletedEndpoint.GetHost()
+	remainingHost := remainingEndpoint.GetHost()
+
+	cm.DeleteEndpoint(config.Name, deletedEndpoint.ID)
+
+	runtime := cm.store.clustersMap[config.Name]
+	if !assert.NotNil(t, runtime) {
+		return
+	}
+	assert.NotSame(t, staleRuntime, runtime)
+	assert.Same(t, config, runtime.Config)
+	if assert.Len(t, config.Endpoints, 1) {
+		assert.Same(t, remainingEndpoint, config.Endpoints[0])
+	}
+
+	hash := config.ConsistentHash.Hash
+	if !assert.NotNil(t, hash) {
+		return
+	}
+	hostList, ok := hash.(interface{ Hosts() []string })
+	if !assert.True(t, ok) {
+		return
+	}
+	hosts := hostList.Hosts()
+	assert.NotContains(t, hosts, deletedHost)
+	assert.Contains(t, hosts, remainingHost)
+}
+
 func TestClusterManager_Race_RoundRobinPickEndpoint(t *testing.T) {
 	cluster := testCluster("race-round-robin", model.LoadBalancerRoundRobin, []*model.Endpoint{
 		testEndpoint("ep-1", "127.0.0.1", 19100),
@@ -195,3 +474,29 @@ func testEndpoint(id string, host string, port int) *model.Endpoint {
 		},
 	}
 }
+
+func testHealthCheck() model.HealthCheckConfig {
+	return model.HealthCheckConfig{
+		Protocol:       "tcp",
+		TimeoutConfig:  "1h",
+		IntervalConfig: "1h",
+	}
+}
+
+func stopStoreRuntimes(store *ClusterStore) {
+	if store == nil {
+		return
+	}
+	for _, runtime := range store.clustersMap {
+		if runtime != nil {
+			runtime.Stop()
+		}
+	}
+}
+
+func healthCheckersLen(runtime *cluster.Cluster) int {
+	if runtime == nil || runtime.HealthCheck == nil {
+		return 0
+	}
+	return reflect.ValueOf(runtime.HealthCheck).Elem().FieldByName("checkers").Len()
+}