You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dubbo.apache.org by xu...@apache.org on 2026/05/06 03:06:14 UTC
(dubbo-go-pixiu) branch develop updated: fix(cluster): tighten runtime consistency (#919)
This is an automated email from the ASF dual-hosted git repository.
Alanxtl pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/dubbo-go-pixiu.git
The following commit(s) were added to refs/heads/develop by this push:
new 55c62109 fix(cluster): tighten runtime consistency (#919)
55c62109 is described below
commit 55c62109a5f73633ce4755cbd0a8c92f4980c5fd
Author: 承潜 <29...@qq.com>
AuthorDate: Wed May 6 11:06:09 2026 +0800
fix(cluster): tighten runtime consistency (#919)
* fix(cluster): tighten runtime consistency
* fix(cluster): address runtime consistency review comments
---
pkg/cluster/healthcheck/healthcheck.go | 25 ++-
pkg/server/cluster_manager.go | 217 ++++++++++++++++++-----
pkg/server/cluster_manager_test.go | 305 +++++++++++++++++++++++++++++++++
3 files changed, 496 insertions(+), 51 deletions(-)
diff --git a/pkg/cluster/healthcheck/healthcheck.go b/pkg/cluster/healthcheck/healthcheck.go
index 5a0d0921..d899bd35 100644
--- a/pkg/cluster/healthcheck/healthcheck.go
+++ b/pkg/cluster/healthcheck/healthcheck.go
@@ -212,13 +212,7 @@ func (hc *HealthChecker) getCheckInterval() time.Duration {
}
func (c *EndpointChecker) Start() {
- defer func() {
- if r := recover(); r != nil {
- logger.Warnf("[health check] node checker panic %v\n%s", r, string(debug.Stack()))
- }
- c.checkTimer.Stop()
- c.checkTimeout.Stop()
- }()
+ defer c.cleanupStart()
c.checkTimer = gxtime.AfterFunc(c.HealthChecker.initialDelay, c.OnCheck)
for {
select {
@@ -256,6 +250,23 @@ func (c *EndpointChecker) Start() {
}
}
+func (c *EndpointChecker) cleanupStart() {
+ if r := recover(); r != nil {
+ logger.Warnf("[health check] node checker panic %v\n%s", r, string(debug.Stack()))
+ }
+ c.stopCheckTimers()
+}
+
+func (c *EndpointChecker) stopCheckTimers() {
+ // Early Stop may run before timers are assigned.
+ if c.checkTimer != nil {
+ c.checkTimer.Stop()
+ }
+ if c.checkTimeout != nil {
+ c.checkTimeout.Stop()
+ }
+}
+
func (c *EndpointChecker) Stop() {
c.once.Do(func() {
close(c.stop)
diff --git a/pkg/server/cluster_manager.go b/pkg/server/cluster_manager.go
index 09b60ba2..8b44ba65 100644
--- a/pkg/server/cluster_manager.go
+++ b/pkg/server/cluster_manager.go
@@ -143,17 +143,35 @@ func (cm *ClusterManager) NewStore(version int32) *ClusterStore {
return &ClusterStore{Version: version, clustersMap: map[string]*cluster.Cluster{}}
}
+// CompareAndSetStore swaps the store only when versions match.
+// Version mismatch must leave both stores and runtime clusters untouched.
func (cm *ClusterManager) CompareAndSetStore(store *ClusterStore) bool {
+ swapped, replacedClusters := cm.compareAndSetStore(store)
+ if !swapped {
+ return false
+ }
+
+ // Stop old runtime after publishing the swap; Stop may touch timers/goroutines.
+ stopClusters(replacedClusters)
+ return true
+}
+
+func (cm *ClusterManager) compareAndSetStore(store *ClusterStore) (bool, []*cluster.Cluster) {
cm.rw.Lock()
defer cm.rw.Unlock()
if store.Version != cm.store.Version {
- return false
+ return false, nil
}
- store.carryOverRuntimeStateFrom(cm.store)
+ currentStore := cm.store
+ replacedClusters := store.ensureRuntimeClusters()
+ store.carryOverRuntimeStateFrom(currentStore)
cm.store = store
- return true
+ if store != currentStore {
+ replacedClusters = append(replacedClusters, currentStore.runtimeClustersNotIn(store)...)
+ }
+ return true, replacedClusters
}
// PickEndpoint picks an endpoint from the cluster by its name and load balancing policy.
@@ -249,7 +267,7 @@ func (cm *ClusterManager) RemoveCluster(namesToDel []string) {
for _, name := range namesToDel { // suppose resource to remove and clusters is few
if name == c.Name {
removed := cm.store.Config[i]
- cm.store.clustersMap[removed.Name].Stop()
+ stopClusters([]*cluster.Cluster{cm.store.clustersMap[removed.Name]})
cm.store.Config[i] = nil
delete(cm.store.clustersMap, removed.Name)
}
@@ -278,10 +296,15 @@ func (s *ClusterStore) AddCluster(c *model.ClusterConfig) {
atomic.AddInt32(&clusterIndex, 1)
}
- s.assembleClusterEndpoints(c)
+ s.prepareClusterConfig(c)
s.Config = append(s.Config, c)
- s.clustersMap[c.Name] = cluster.NewCluster(c)
+ stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(c.Name, c)})
+}
+
+// prepareClusterConfig rebuilds endpoint defaults and hash from current endpoints.
+func (s *ClusterStore) prepareClusterConfig(c *model.ClusterConfig) {
+ s.assembleClusterEndpoints(c)
c.CreateConsistentHash()
}
@@ -309,10 +332,108 @@ func (s *ClusterStore) assembleClusterEndpoints(c *model.ClusterConfig) {
}
}
+// replaceClusterRuntime returns the old runtime so callers decide when to stop it.
+func (s *ClusterStore) replaceClusterRuntime(name string, config *model.ClusterConfig) *cluster.Cluster {
+ if s.clustersMap == nil {
+ s.clustersMap = map[string]*cluster.Cluster{}
+ }
+
+ oldRuntime := s.clustersMap[name]
+ s.clustersMap[name] = cluster.NewCluster(config)
+ return oldRuntime
+}
+
+// ensureRuntimeClusters repairs clustersMap to match Config by name and pointer.
+func (s *ClusterStore) ensureRuntimeClusters() []*cluster.Cluster {
+ if s == nil {
+ return nil
+ }
+ if s.clustersMap == nil {
+ s.clustersMap = map[string]*cluster.Cluster{}
+ }
+
+ replacedClusters := make([]*cluster.Cluster, 0)
+ configsByName := make(map[string]*model.ClusterConfig, len(s.Config))
+ for _, clusterConfig := range s.Config {
+ if clusterConfig == nil {
+ continue
+ }
+ s.prepareClusterConfig(clusterConfig)
+ configsByName[clusterConfig.Name] = clusterConfig
+
+ runtimeCluster := s.clustersMap[clusterConfig.Name]
+ if runtimeCluster == nil || runtimeCluster.Config != clusterConfig {
+ if oldRuntime := s.replaceClusterRuntime(clusterConfig.Name, clusterConfig); oldRuntime != nil {
+ replacedClusters = append(replacedClusters, oldRuntime)
+ }
+ }
+ }
+
+ for name, runtimeCluster := range s.clustersMap {
+ if _, ok := configsByName[name]; !ok {
+ replacedClusters = append(replacedClusters, runtimeCluster)
+ delete(s.clustersMap, name)
+ }
+ }
+ return replacedClusters
+}
+
+// runtimeClustersNotIn finds old runtime clusters dropped by the next store.
+func (s *ClusterStore) runtimeClustersNotIn(next *ClusterStore) []*cluster.Cluster {
+ if s == nil {
+ return nil
+ }
+
+ nextClusters := make(map[*cluster.Cluster]struct{})
+ if next != nil {
+ for _, runtimeCluster := range next.clustersMap {
+ if runtimeCluster != nil {
+ nextClusters[runtimeCluster] = struct{}{}
+ }
+ }
+ }
+
+ replacedClusters := make([]*cluster.Cluster, 0)
+ for _, runtimeCluster := range s.clustersMap {
+ if runtimeCluster == nil {
+ continue
+ }
+ if _, ok := nextClusters[runtimeCluster]; !ok {
+ replacedClusters = append(replacedClusters, runtimeCluster)
+ }
+ }
+ return replacedClusters
+}
+
+// stopClusters is nil-safe and avoids stopping the same runtime twice.
+func stopClusters(clusters []*cluster.Cluster) {
+ stopped := make(map[*cluster.Cluster]struct{}, len(clusters))
+ for _, runtimeCluster := range clusters {
+ if runtimeCluster == nil {
+ continue
+ }
+ if _, ok := stopped[runtimeCluster]; ok {
+ continue
+ }
+ stopped[runtimeCluster] = struct{}{}
+ runtimeCluster.Stop()
+ }
+}
+
+// UpdateCluster replaces config/runtime together while preserving the RR cursor.
func (s *ClusterStore) UpdateCluster(new *model.ClusterConfig) {
for i, c := range s.Config {
+ if c == nil {
+ continue
+ }
if c.Name == new.Name {
+ s.prepareClusterConfig(new)
+ atomic.StoreUint32(
+ &new.PrePickEndpointIndex,
+ atomic.LoadUint32(&c.PrePickEndpointIndex),
+ )
s.Config[i] = new
+ stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(new.Name, new)})
return
}
}
@@ -320,59 +441,67 @@ func (s *ClusterStore) UpdateCluster(new *model.ClusterConfig) {
}
func (s *ClusterStore) SetEndpoint(clusterName string, endpoint *model.Endpoint) {
- cluster := s.clustersMap[clusterName]
- if cluster == nil {
+ clusterConfig := s.findClusterConfig(clusterName)
+ if clusterConfig == nil {
c := &model.ClusterConfig{Name: clusterName, LbStr: model.LoadBalancerRoundRobin, Endpoints: []*model.Endpoint{}}
s.AddCluster(c)
- cluster = s.clustersMap[clusterName]
+ clusterConfig = c
}
- for _, c := range s.Config {
- if c.Name == clusterName {
- for _, e := range c.Endpoints {
- // endpoint update
- if e.ID == endpoint.ID {
- cluster.RemoveEndpoint(e)
- e.Name = endpoint.Name
- e.Metadata = endpoint.Metadata
- e.Address = endpoint.Address
- cluster.AddEndpoint(e)
- return
- }
- }
- // endpoint create
- c.Endpoints = append(c.Endpoints, endpoint)
- cluster.AddEndpoint(endpoint)
- if c.ConsistentHash.Hash != nil {
- c.ConsistentHash.Hash.Add(endpoint.GetHost())
- }
+ runtimeCluster := s.clustersMap[clusterName]
+ if runtimeCluster == nil || runtimeCluster.Config != clusterConfig {
+ stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(clusterName, clusterConfig)})
+ runtimeCluster = s.clustersMap[clusterName]
+ }
+
+ for _, e := range clusterConfig.Endpoints {
+ if e.ID == endpoint.ID {
+ // Remove before mutating address because healthcheck keys by address.
+ runtimeCluster.RemoveEndpoint(e)
+ e.Name = endpoint.Name
+ e.Metadata = endpoint.Metadata
+ e.Address = endpoint.Address
+ s.prepareClusterConfig(clusterConfig)
+ runtimeCluster.AddEndpoint(e)
return
}
}
+ clusterConfig.Endpoints = append(clusterConfig.Endpoints, endpoint)
+ s.prepareClusterConfig(clusterConfig)
+ runtimeCluster.AddEndpoint(endpoint)
}
func (s *ClusterStore) DeleteEndpoint(clusterName string, endpointID string) {
- cluster := s.clustersMap[clusterName]
- if cluster == nil {
+ clusterConfig := s.findClusterConfig(clusterName)
+ if clusterConfig == nil {
+ logger.Warnf("not found cluster %s", clusterName)
return
}
- for _, c := range s.Config {
- if c.Name == clusterName {
- for i, e := range c.Endpoints {
- if e.ID == endpointID {
- cluster.RemoveEndpoint(e)
- c.Endpoints = append(c.Endpoints[:i], c.Endpoints[i+1:]...)
- if c.ConsistentHash.Hash != nil {
- c.ConsistentHash.Hash.Remove(e.GetHost())
- }
- return
- }
- }
- logger.Warnf("not found endpoint %s", endpointID)
+
+ runtimeCluster := s.clustersMap[clusterName]
+ if runtimeCluster == nil || runtimeCluster.Config != clusterConfig {
+ stopClusters([]*cluster.Cluster{s.replaceClusterRuntime(clusterName, clusterConfig)})
+ runtimeCluster = s.clustersMap[clusterName]
+ }
+
+ for i, e := range clusterConfig.Endpoints {
+ if e.ID == endpointID {
+ runtimeCluster.RemoveEndpoint(e)
+ clusterConfig.Endpoints = append(clusterConfig.Endpoints[:i], clusterConfig.Endpoints[i+1:]...)
+ s.prepareClusterConfig(clusterConfig)
return
}
}
- logger.Warnf("not found cluster %s", clusterName)
+ logger.Warnf("not found endpoint %s", endpointID)
+}
+
+func (s *ClusterStore) findClusterConfig(clusterName string) *model.ClusterConfig {
+ for _, c := range s.Config {
+ if c != nil && c.Name == clusterName {
+ return c
+ }
+ }
+ return nil
}
func (s *ClusterStore) HasCluster(clusterName string) bool {
diff --git a/pkg/server/cluster_manager_test.go b/pkg/server/cluster_manager_test.go
index dd06e25a..6faf6cb9 100644
--- a/pkg/server/cluster_manager_test.go
+++ b/pkg/server/cluster_manager_test.go
@@ -19,6 +19,7 @@ package server
import (
"fmt"
+ "reflect"
"sync"
"sync/atomic"
"testing"
@@ -29,6 +30,7 @@ import (
)
import (
+ "github.com/apache/dubbo-go-pixiu/pkg/cluster"
_ "github.com/apache/dubbo-go-pixiu/pkg/cluster/loadbalancer/maglev" // Register Maglev for cluster-manager tests.
_ "github.com/apache/dubbo-go-pixiu/pkg/cluster/loadbalancer/rand" // Register Rand for cluster-manager tests.
_ "github.com/apache/dubbo-go-pixiu/pkg/cluster/loadbalancer/ringhash" // Register RingHash for cluster-manager tests.
@@ -138,6 +140,283 @@ func TestClusterManager_CompareAndSetStorePreservesRoundRobinCursorAcrossRefresh
}
}
+func TestClusterManager_UpdateClusterRebuildsRuntimeCluster(t *testing.T) {
+ oldConfig := testCluster("runtime-update", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-1", "127.0.0.1", 19300),
+ }, testHealthCheck())
+ cm := testClusterManager(oldConfig)
+ defer stopStoreRuntimes(cm.store)
+
+ oldRuntime := cm.store.clustersMap[oldConfig.Name]
+ if !assert.NotNil(t, oldRuntime) {
+ return
+ }
+ assert.Same(t, oldConfig, oldRuntime.Config)
+ assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+ const expectedCursor uint32 = 11
+ atomic.StoreUint32(&oldConfig.PrePickEndpointIndex, expectedCursor)
+
+ newConfig := testCluster(oldConfig.Name, model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-2", "127.0.0.1", 19301),
+ }, testHealthCheck())
+
+ cm.UpdateCluster(newConfig)
+
+ newRuntime := cm.store.clustersMap[oldConfig.Name]
+ if !assert.NotNil(t, newRuntime) {
+ return
+ }
+ assert.NotSame(t, oldRuntime, newRuntime)
+ assert.Same(t, newConfig, newRuntime.Config)
+ assert.Same(t, newConfig, cm.store.Config[0])
+ assert.Equal(t, expectedCursor, atomic.LoadUint32(&newConfig.PrePickEndpointIndex))
+ assert.Equal(t, 0, healthCheckersLen(oldRuntime))
+ assert.Greater(t, healthCheckersLen(newRuntime), 0)
+}
+
+func TestClusterManager_CompareAndSetStoreVersionMismatchHasNoSideEffects(t *testing.T) {
+ currentConfig := testCluster("cas-version-mismatch", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-1", "127.0.0.1", 19310),
+ }, testHealthCheck())
+ cm := testClusterManager(currentConfig)
+ defer stopStoreRuntimes(cm.store)
+
+ oldStore := cm.store
+ oldRuntime := oldStore.clustersMap[currentConfig.Name]
+ if !assert.NotNil(t, oldRuntime) {
+ return
+ }
+ assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+ candidateConfig := testCluster(currentConfig.Name, model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-2", "127.0.0.1", 19311),
+ })
+ candidate := &ClusterStore{
+ Config: []*model.ClusterConfig{candidateConfig},
+ Version: oldStore.Version + 1,
+ }
+
+ assert.False(t, cm.CompareAndSetStore(candidate))
+ assert.Same(t, oldStore, cm.store)
+ assert.Nil(t, candidate.clustersMap)
+ assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+}
+
+func TestClusterManager_CompareAndSetStoreEnsuresRuntimeAndStopsOld(t *testing.T) {
+ oldConfig := testCluster("cas-runtime-refresh", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-1", "127.0.0.1", 19320),
+ }, testHealthCheck())
+ cm := testClusterManager(oldConfig)
+
+ oldRuntime := cm.store.clustersMap[oldConfig.Name]
+ if !assert.NotNil(t, oldRuntime) {
+ return
+ }
+ assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+ const expectedCursor uint32 = 17
+ atomic.StoreUint32(&oldConfig.PrePickEndpointIndex, expectedCursor)
+
+ newConfig := testCluster(oldConfig.Name, model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-2", "127.0.0.1", 19321),
+ }, testHealthCheck())
+ candidate := &ClusterStore{
+ Config: []*model.ClusterConfig{newConfig},
+ Version: cm.store.Version,
+ }
+
+ assert.True(t, cm.CompareAndSetStore(candidate))
+ defer stopStoreRuntimes(cm.store)
+
+ newRuntime := candidate.clustersMap[newConfig.Name]
+ if !assert.NotNil(t, newRuntime) {
+ return
+ }
+ assert.Same(t, candidate, cm.store)
+ assert.NotSame(t, oldRuntime, newRuntime)
+ assert.Same(t, newConfig, newRuntime.Config)
+ assert.Equal(t, expectedCursor, atomic.LoadUint32(&newConfig.PrePickEndpointIndex))
+ assert.Equal(t, 0, healthCheckersLen(oldRuntime))
+ assert.Greater(t, healthCheckersLen(newRuntime), 0)
+}
+
+func TestClusterManager_CompareAndSetStoreStopsRemovedRuntime(t *testing.T) {
+ oldConfig := testCluster("cas-runtime-removed", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-1", "127.0.0.1", 19325),
+ }, testHealthCheck())
+ cm := testClusterManager(oldConfig)
+
+ oldRuntime := cm.store.clustersMap[oldConfig.Name]
+ if !assert.NotNil(t, oldRuntime) {
+ return
+ }
+ assert.Greater(t, healthCheckersLen(oldRuntime), 0)
+
+ candidate := &ClusterStore{
+ Version: cm.store.Version,
+ }
+
+ assert.True(t, cm.CompareAndSetStore(candidate))
+ assert.Same(t, candidate, cm.store)
+ assert.Empty(t, cm.store.Config)
+ assert.NotContains(t, cm.store.clustersMap, oldConfig.Name)
+ assert.Equal(t, 0, healthCheckersLen(oldRuntime))
+}
+
+func TestClusterStore_EnsureRuntimeClustersRepairsRuntimeMap(t *testing.T) {
+ t.Run("nil map", func(t *testing.T) {
+ config := testCluster("ensure-nil-map", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-1", "127.0.0.1", 19330),
+ })
+ store := &ClusterStore{Config: []*model.ClusterConfig{config}}
+
+ replaced := store.ensureRuntimeClusters()
+ defer stopStoreRuntimes(store)
+
+ assert.Empty(t, replaced)
+ if assert.NotNil(t, store.clustersMap[config.Name]) {
+ assert.Same(t, config, store.clustersMap[config.Name].Config)
+ }
+ })
+
+ t.Run("missing mismatched stale and idempotent", func(t *testing.T) {
+ correctConfig := testCluster("ensure-correct", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-1", "127.0.0.1", 19331),
+ })
+ missingConfig := testCluster("ensure-missing", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("ep-2", "127.0.0.1", 19332),
+ })
+ oldMismatchedConfig := testCluster("ensure-mismatch", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("old", "127.0.0.1", 19333),
+ })
+ newMismatchedConfig := testCluster("ensure-mismatch", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("new", "127.0.0.1", 19334),
+ })
+ staleConfig := testCluster("ensure-stale", model.LoadBalancerRoundRobin, []*model.Endpoint{
+ testEndpoint("stale", "127.0.0.1", 19335),
+ })
+
+ correctRuntime := cluster.NewCluster(correctConfig)
+ mismatchedRuntime := cluster.NewCluster(oldMismatchedConfig)
+ staleRuntime := cluster.NewCluster(staleConfig)
+ store := &ClusterStore{
+ Config: []*model.ClusterConfig{
+ correctConfig,
+ missingConfig,
+ newMismatchedConfig,
+ },
+ clustersMap: map[string]*cluster.Cluster{
+ correctConfig.Name: correctRuntime,
+ newMismatchedConfig.Name: mismatchedRuntime,
+ staleConfig.Name: staleRuntime,
+ },
+ }
+ defer stopStoreRuntimes(store)
+
+ replaced := store.ensureRuntimeClusters()
+ stopClusters(replaced)
+
+ assert.Same(t, correctRuntime, store.clustersMap[correctConfig.Name])
+ if assert.NotNil(t, store.clustersMap[missingConfig.Name]) {
+ assert.Same(t, missingConfig, store.clustersMap[missingConfig.Name].Config)
+ }
+ if assert.NotNil(t, store.clustersMap[newMismatchedConfig.Name]) {
+ assert.NotSame(t, mismatchedRuntime, store.clustersMap[newMismatchedConfig.Name])
+ assert.Same(t, newMismatchedConfig, store.clustersMap[newMismatchedConfig.Name].Config)
+ }
+ assert.NotContains(t, store.clustersMap, staleConfig.Name)
+ assert.Contains(t, replaced, mismatchedRuntime)
+ assert.Contains(t, replaced, staleRuntime)
+
+ runtimesAfterRepair := map[string]*cluster.Cluster{}
+ for name, runtime := range store.clustersMap {
+ runtimesAfterRepair[name] = runtime
+ }
+
+ assert.Empty(t, store.ensureRuntimeClusters())
+ assert.Equal(t, runtimesAfterRepair, store.clustersMap)
+ })
+}
+
+func TestClusterManager_SetEndpointUpdateRebuildsConsistentHash(t *testing.T) {
+ tests := []model.LbPolicyType{
+ model.LoadBalancerRingHashing,
+ model.LoadBalancerMaglevHashing,
+ }
+
+ for _, lb := range tests {
+ t.Run(string(lb), func(t *testing.T) {
+ oldEndpoint := testEndpoint("ep-1", "127.0.0.1", 19340)
+ oldHost := oldEndpoint.GetHost()
+ config := testCluster(fmt.Sprintf("hash-update-%s", lb), lb, []*model.Endpoint{oldEndpoint})
+ cm := testClusterManager(config)
+ defer stopStoreRuntimes(cm.store)
+
+ newEndpoint := testEndpoint("ep-1", "127.0.0.2", 19341)
+ newHost := newEndpoint.GetHost()
+ cm.SetEndpoint(config.Name, newEndpoint)
+
+ hash := cm.store.Config[0].ConsistentHash.Hash
+ if !assert.NotNil(t, hash) {
+ return
+ }
+ if hostList, ok := hash.(interface{ Hosts() []string }); ok {
+ hosts := hostList.Hosts()
+ assert.NotContains(t, hosts, oldHost)
+ assert.Contains(t, hosts, newHost)
+ return
+ }
+ assert.False(t, hash.Remove(oldHost))
+ assert.True(t, hash.Remove(newHost))
+ })
+ }
+}
+
+func TestClusterManager_DeleteEndpointRepairsRuntimeAndConsistentHash(t *testing.T) {
+ deletedEndpoint := testEndpoint("ep-1", "127.0.0.1", 19350)
+ remainingEndpoint := testEndpoint("ep-2", "127.0.0.1", 19351)
+ config := testCluster("delete-runtime-repair", model.LoadBalancerRingHashing, []*model.Endpoint{
+ deletedEndpoint,
+ remainingEndpoint,
+ })
+ cm := testClusterManager(config)
+ defer stopStoreRuntimes(cm.store)
+
+ staleConfig := testCluster(config.Name, model.LoadBalancerRingHashing, []*model.Endpoint{
+ testEndpoint("stale", "127.0.0.1", 19352),
+ })
+ staleRuntime := cluster.NewCluster(staleConfig)
+ cm.store.clustersMap[config.Name] = staleRuntime
+
+ deletedHost := deletedEndpoint.GetHost()
+ remainingHost := remainingEndpoint.GetHost()
+
+ cm.DeleteEndpoint(config.Name, deletedEndpoint.ID)
+
+ runtime := cm.store.clustersMap[config.Name]
+ if !assert.NotNil(t, runtime) {
+ return
+ }
+ assert.NotSame(t, staleRuntime, runtime)
+ assert.Same(t, config, runtime.Config)
+ if assert.Len(t, config.Endpoints, 1) {
+ assert.Same(t, remainingEndpoint, config.Endpoints[0])
+ }
+
+ hash := config.ConsistentHash.Hash
+ if !assert.NotNil(t, hash) {
+ return
+ }
+ hostList, ok := hash.(interface{ Hosts() []string })
+ if !assert.True(t, ok) {
+ return
+ }
+ hosts := hostList.Hosts()
+ assert.NotContains(t, hosts, deletedHost)
+ assert.Contains(t, hosts, remainingHost)
+}
+
func TestClusterManager_Race_RoundRobinPickEndpoint(t *testing.T) {
cluster := testCluster("race-round-robin", model.LoadBalancerRoundRobin, []*model.Endpoint{
testEndpoint("ep-1", "127.0.0.1", 19100),
@@ -195,3 +474,29 @@ func testEndpoint(id string, host string, port int) *model.Endpoint {
},
}
}
+
+func testHealthCheck() model.HealthCheckConfig {
+ return model.HealthCheckConfig{
+ Protocol: "tcp",
+ TimeoutConfig: "1h",
+ IntervalConfig: "1h",
+ }
+}
+
+func stopStoreRuntimes(store *ClusterStore) {
+ if store == nil {
+ return
+ }
+ for _, runtime := range store.clustersMap {
+ if runtime != nil {
+ runtime.Stop()
+ }
+ }
+}
+
+func healthCheckersLen(runtime *cluster.Cluster) int {
+ if runtime == nil || runtime.HealthCheck == nil {
+ return 0
+ }
+ return reflect.ValueOf(runtime.HealthCheck).Elem().FieldByName("checkers").Len()
+}