You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by ya...@apache.org on 2020/03/09 16:15:00 UTC

[servicecomb-java-chassis] 11/19: [SCB-1691] RegistryUtils manage multiple ServiceRegistry instances

This is an automated email from the ASF dual-hosted git repository.

yaohaishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-java-chassis.git

commit 7828f7fbe923d42c78cab79d2ab78ed85033f9c0
Author: yhs0092 <yh...@163.com>
AuthorDate: Tue Feb 11 00:17:51 2020 +0800

    [SCB-1691] RegistryUtils manage multiple ServiceRegistry instances
---
 .../loadbalance/TestLoadBalanceHandler2.java       |  18 ++--
 .../servicecomb/serviceregistry/RegistryUtils.java | 120 +++++++++++++++++++--
 .../cache/MicroserviceInstanceCache.java           |   8 +-
 .../serviceregistry/MockMicroserviceVersions.java  |   8 ++
 .../servicecomb/serviceregistry/TestConsumers.java |  17 ---
 .../servicecomb/serviceregistry/TestRegistry.java  |   9 +-
 .../serviceregistry/TestRegistryBase.java          |  24 ++---
 .../cache/TestMicroserviceInstanceCache.java       |  35 ++++--
 .../instance/TestInstanceCacheChecker.java         |   2 +-
 .../springboot/common/AbstractDiscoveryClient.java |  26 ++---
 10 files changed, 191 insertions(+), 76 deletions(-)

diff --git a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java
index 2edc5bd..36ee88a 100644
--- a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java
+++ b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java
@@ -119,7 +119,7 @@ public class TestLoadBalanceHandler2 {
     Invocation invocation = new Invocation(referenceConfig, operationMeta, new HashMap<>());
 
     InstanceCacheManager instanceCacheManager = Mockito.mock(InstanceCacheManager.class);
-    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    ServiceRegistry serviceRegistry = mockUpServiceRegistry();
     TransportManager transportManager = Mockito.mock(TransportManager.class);
     Transport transport = Mockito.mock(Transport.class);
     ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.operation.enabled", "false");
@@ -268,7 +268,7 @@ public class TestLoadBalanceHandler2 {
     Invocation invocation = new Invocation(referenceConfig, operationMeta, new HashMap<>());
 
     InstanceCacheManager instanceCacheManager = Mockito.mock(InstanceCacheManager.class);
-    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    ServiceRegistry serviceRegistry = mockUpServiceRegistry();
     TransportManager transportManager = Mockito.mock(TransportManager.class);
     Transport transport = Mockito.mock(Transport.class);
     ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.operation.enabled", "false");
@@ -357,7 +357,7 @@ public class TestLoadBalanceHandler2 {
     Invocation invocation = new Invocation(referenceConfig, operationMeta, new HashMap<>());
 
     InstanceCacheManager instanceCacheManager = Mockito.mock(InstanceCacheManager.class);
-    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    ServiceRegistry serviceRegistry = mockUpServiceRegistry();
     TransportManager transportManager = Mockito.mock(TransportManager.class);
     Transport transport = Mockito.mock(Transport.class);
     ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.operation.enabled", "false");
@@ -484,7 +484,7 @@ public class TestLoadBalanceHandler2 {
     });
 
     InstanceCacheManager instanceCacheManager = Mockito.mock(InstanceCacheManager.class);
-    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    ServiceRegistry serviceRegistry = mockUpServiceRegistry();
     TransportManager transportManager = Mockito.mock(TransportManager.class);
     Transport transport = Mockito.mock(Transport.class);
     ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.operation.enabled", "false");
@@ -627,7 +627,7 @@ public class TestLoadBalanceHandler2 {
     });
 
     InstanceCacheManager instanceCacheManager = Mockito.mock(InstanceCacheManager.class);
-    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    ServiceRegistry serviceRegistry = mockUpServiceRegistry();
     TransportManager transportManager = Mockito.mock(TransportManager.class);
     Transport transport = Mockito.mock(Transport.class);
     ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.operation.enabled", "false");
@@ -778,7 +778,7 @@ public class TestLoadBalanceHandler2 {
     AsyncResponse asyncResp = Mockito.mock(AsyncResponse.class);
 
     InstanceCacheManager instanceCacheManager = Mockito.mock(InstanceCacheManager.class);
-    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    ServiceRegistry serviceRegistry = mockUpServiceRegistry();
     TransportManager transportManager = Mockito.mock(TransportManager.class);
     Transport transport = Mockito.mock(Transport.class);
     ArchaiusUtils.setProperty("servicecomb.loadbalance.filter.operation.enabled", "false");
@@ -992,4 +992,10 @@ public class TestLoadBalanceHandler2 {
       }
     };
   }
+
+  private ServiceRegistry mockUpServiceRegistry() {
+    ServiceRegistry serviceRegistry = Mockito.mock(ServiceRegistry.class);
+    when(serviceRegistry.getEventBus()).thenReturn(EventManager.getEventBus());
+    return serviceRegistry;
+  }
 }
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/RegistryUtils.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/RegistryUtils.java
index 2f93bd6..49f28e2 100644
--- a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/RegistryUtils.java
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/RegistryUtils.java
@@ -21,14 +21,18 @@ import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.regex.Matcher;
 
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.servicecomb.config.ConfigUtil;
 import org.apache.servicecomb.config.archaius.sources.MicroserviceConfigLoader;
+import org.apache.servicecomb.foundation.common.Holder;
 import org.apache.servicecomb.foundation.common.event.EventManager;
 import org.apache.servicecomb.foundation.common.net.IpPort;
 import org.apache.servicecomb.foundation.common.net.NetUtils;
@@ -43,7 +47,10 @@ import org.apache.servicecomb.serviceregistry.config.ServiceRegistryConfig;
 import org.apache.servicecomb.serviceregistry.consumer.AppManager;
 import org.apache.servicecomb.serviceregistry.definition.MicroserviceDefinition;
 import org.apache.servicecomb.serviceregistry.registry.ServiceRegistryFactory;
+import org.apache.servicecomb.serviceregistry.registry.cache.AggregateServiceRegistryCache;
 import org.apache.servicecomb.serviceregistry.registry.cache.MicroserviceCache;
+import org.apache.servicecomb.serviceregistry.registry.cache.MicroserviceCache.MicroserviceCacheStatus;
+import org.apache.servicecomb.serviceregistry.registry.cache.MicroserviceCacheKey;
 import org.apache.servicecomb.serviceregistry.swagger.SwaggerLoader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,6 +62,9 @@ import com.netflix.config.DynamicPropertyFactory;
 public final class RegistryUtils {
   private static final Logger LOGGER = LoggerFactory.getLogger(RegistryUtils.class);
 
+  /**
+   * The default ServiceRegistry instance
+   */
   private static volatile ServiceRegistry serviceRegistry;
 
   // value is ip or {interface name}
@@ -68,6 +78,12 @@ public final class RegistryUtils {
 
   private static InstanceCacheManager instanceCacheManager = new InstanceCacheManagerNew(appManager);
 
+  private static final Map<String, ServiceRegistryConfig> EXTRA_SERVICE_REGISTRY_CONFIGS = new LinkedHashMap<>();
+
+  private static final Map<String, ServiceRegistry> EXTRA_SERVICE_REGISTRIES = new LinkedHashMap<>();
+
+  static AggregateServiceRegistryCache aggregateServiceRegistryCache;
+
   private RegistryUtils() {
   }
 
@@ -78,18 +94,43 @@ public final class RegistryUtils {
 
     MicroserviceConfigLoader loader = ConfigUtil.getMicroserviceConfigLoader();
     MicroserviceDefinition microserviceDefinition = new MicroserviceDefinition(loader.getConfigModels());
+    initializeServiceRegistries(microserviceDefinition);
+
+    initAggregateServiceRegistryCache();
+  }
+
+  private static void initAggregateServiceRegistryCache() {
+    ArrayList<ServiceRegistry> serviceRegistries = new ArrayList<>();
+    executeOnEachServiceRegistry(serviceRegistries::add);
+    aggregateServiceRegistryCache = new AggregateServiceRegistryCache(serviceRegistries);
+    aggregateServiceRegistryCache.setCacheRefreshedWatcher(refreshedCaches -> {
+      appManager.pullInstances();
+    });
+
+    executeOnEachServiceRegistry(
+        serviceRegistry -> serviceRegistry
+            .getEventBus()
+            .register(aggregateServiceRegistryCache));
+  }
+
+  private static void initializeServiceRegistries(MicroserviceDefinition microserviceDefinition) {
     serviceRegistry =
         ServiceRegistryFactory
             .create(EventManager.eventBus, ServiceRegistryConfig.INSTANCE, microserviceDefinition);
-    serviceRegistry.init();
+    EXTRA_SERVICE_REGISTRY_CONFIGS.forEach((k, v) -> {
+      ServiceRegistry serviceRegistry = ServiceRegistryFactory
+          .create(EventManager.getEventBus(), v, microserviceDefinition);
+      addExtraServiceRegistry(serviceRegistry);
+    });
+    executeOnEachServiceRegistry(ServiceRegistry::init);
   }
 
   public static void run() {
-    serviceRegistry.run();
+    executeOnEachServiceRegistry(ServiceRegistry::run);
   }
 
   public static void destroy() {
-    serviceRegistry.destroy();
+    executeOnEachServiceRegistry(ServiceRegistry::destroy);
   }
 
   /**
@@ -106,8 +147,10 @@ public final class RegistryUtils {
 
   public static void setServiceRegistry(ServiceRegistry serviceRegistry) {
     RegistryUtils.serviceRegistry = serviceRegistry;
+    initAggregateServiceRegistryCache();
   }
 
+  @Deprecated
   public static ServiceRegistryClient getServiceRegistryClient() {
     return serviceRegistry.getServiceRegistryClient();
   }
@@ -236,21 +279,35 @@ public final class RegistryUtils {
 
   public static List<MicroserviceInstance> findServiceInstance(String appId, String serviceName,
       String versionRule) {
-    return serviceRegistry.findServiceInstance(appId, serviceName, versionRule);
+    MicroserviceCache serviceCache = aggregateServiceRegistryCache.findServiceCache(
+        MicroserviceCacheKey.builder()
+            .appId(appId).serviceName(serviceName).env(getMicroservice().getEnvironment())
+            .build()
+    );
+    return MicroserviceCacheStatus.SERVICE_NOT_FOUND.equals(serviceCache.getStatus()) ?
+        null : serviceCache.getInstances();
   }
 
   // update microservice instance properties
   public static boolean updateInstanceProperties(Map<String, String> instanceProperties) {
-    return serviceRegistry.updateInstanceProperties(instanceProperties);
+    Holder<Boolean> resultHolder = new Holder<>(true);
+    executeOnEachServiceRegistry(sr -> {
+      boolean updateResult = sr.updateInstanceProperties(instanceProperties);
+      resultHolder.value = updateResult && resultHolder.value;
+    });
+    return resultHolder.value;
   }
 
   public static Microservice getMicroservice(String microserviceId) {
-    return serviceRegistry.getRemoteMicroservice(microserviceId);
+    return getResultFromFirstValidServiceRegistry(sr -> sr.getRemoteMicroservice(microserviceId));
   }
 
   public static MicroserviceInstances findServiceInstances(String appId, String serviceName,
       String versionRule, String revision) {
-    return serviceRegistry.findServiceInstances(appId, serviceName, versionRule, revision);
+    MicroserviceCache serviceCache = aggregateServiceRegistryCache.findServiceCache(
+        MicroserviceCacheKey.builder().appId(appId).serviceName(serviceName).env(getMicroservice().getEnvironment())
+            .build());
+    return convertCacheToMicroserviceInstances(serviceCache);
   }
 
   /**
@@ -288,11 +345,56 @@ public final class RegistryUtils {
   }
 
   public static String getAggregatedSchema(String microserviceId, String schemaId) {
-    return serviceRegistry.getServiceRegistryClient().getAggregatedSchema(microserviceId, schemaId);
+    return getResultFromFirstValidServiceRegistry(
+        sr -> sr.getServiceRegistryClient().getAggregatedSchema(microserviceId, schemaId));
   }
 
   public static Microservice getAggregatedRemoteMicroservice(String microserviceId) {
-    return serviceRegistry.getAggregatedRemoteMicroservice(microserviceId);
+    return getResultFromFirstValidServiceRegistry(
+        sr -> sr.getAggregatedRemoteMicroservice(microserviceId));
+  }
+
+  public static <T> T getResultFromFirstValidServiceRegistry(Function<ServiceRegistry, T> action) {
+    Holder<T> resultHolder = new Holder<>();
+    executeOnEachServiceRegistry(sr -> {
+      if (null == resultHolder.value) {
+        resultHolder.value = action.apply(sr);
+      }
+    });
+    return resultHolder.value;
+  }
+
+  public static void executeOnEachServiceRegistry(Consumer<ServiceRegistry> action) {
+    if (null != getServiceRegistry()) {
+      action.accept(getServiceRegistry());
+    }
+    if (!EXTRA_SERVICE_REGISTRIES.isEmpty()) {
+      EXTRA_SERVICE_REGISTRIES.forEach((k, v) -> action.accept(v));
+    }
+  }
+
+  public static void addExtraServiceRegistry(ServiceRegistry serviceRegistry) {
+    Objects.requireNonNull(serviceRegistry);
+    LOGGER.info("extra ServiceRegistry added: [{}], [{}]", serviceRegistry.getName(), serviceRegistry.getClass());
+    EXTRA_SERVICE_REGISTRIES.put(serviceRegistry.getName(), serviceRegistry);
+  }
+
+  /**
+   * Add the configuration object of {@link ServiceRegistry}.
+   * The corresponding {@link ServiceRegistry} instances are instantiated later in {@link #init()}
+   */
+  public static void addExtraServiceRegistryConfig(ServiceRegistryConfig serviceRegistryConfig) {
+    validateRegistryConfig(serviceRegistryConfig);
+    EXTRA_SERVICE_REGISTRY_CONFIGS.put(serviceRegistryConfig.getRegistryName(), serviceRegistryConfig);
+  }
+
+  /**
+   * @throws NullPointerException serviceRegistryConfig is null
+   * @throws IllegalArgumentException config value is illegal
+   */
+  public static void validateRegistryConfig(ServiceRegistryConfig serviceRegistryConfig) {
+    Objects.requireNonNull(serviceRegistryConfig);
+    validateRegistryName(serviceRegistryConfig.getRegistryName());
   }
 
   /**
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/cache/MicroserviceInstanceCache.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/cache/MicroserviceInstanceCache.java
index 16a0381..80c8fb5 100644
--- a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/cache/MicroserviceInstanceCache.java
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/cache/MicroserviceInstanceCache.java
@@ -52,7 +52,7 @@ public class MicroserviceInstanceCache {
   public static Microservice getOrCreate(String serviceId) {
     try {
       return microservices.get(serviceId, () -> {
-        Microservice microservice = RegistryUtils.getServiceRegistryClient().getAggregatedMicroservice(serviceId);
+        Microservice microservice = RegistryUtils.getAggregatedRemoteMicroservice(serviceId);
         if (microservice == null) {
           throw new IllegalArgumentException("service id not exists.");
         }
@@ -70,9 +70,9 @@ public class MicroserviceInstanceCache {
       return instances.get(key, new Callable<MicroserviceInstance>() {
 
         @Override
-        public MicroserviceInstance call() throws Exception {
-          MicroserviceInstance instance = RegistryUtils.getServiceRegistryClient()
-              .findServiceInstance(serviceId, instanceId);
+        public MicroserviceInstance call() {
+          MicroserviceInstance instance = RegistryUtils.getResultFromFirstValidServiceRegistry(
+              sr -> sr.getServiceRegistryClient().findServiceInstance(serviceId, instanceId));
           if (instance == null) {
             throw new IllegalArgumentException("instance id not exists.");
           }
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/MockMicroserviceVersions.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/MockMicroserviceVersions.java
index 3d183b1..b7dc262 100644
--- a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/MockMicroserviceVersions.java
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/MockMicroserviceVersions.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import org.apache.servicecomb.foundation.common.event.EventManager;
 import org.apache.servicecomb.serviceregistry.api.registry.Microservice;
 import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
 import org.apache.servicecomb.serviceregistry.consumer.AppManager;
@@ -34,6 +35,8 @@ import org.apache.servicecomb.serviceregistry.version.Version;
 import org.hamcrest.Matchers;
 import org.junit.Assert;
 
+import com.google.common.eventbus.EventBus;
+
 import mockit.Mock;
 import mockit.MockUp;
 
@@ -51,6 +54,11 @@ public class MockMicroserviceVersions extends MicroserviceVersions {
       Microservice getAggregatedRemoteMicroservice(String microserviceId) {
         return mockedMicroservices.get(microserviceId);
       }
+
+      @Mock
+      EventBus getEventBus() {
+        return EventManager.getEventBus();
+      }
     }.getMockInstance();
 
     RegistryUtils.setServiceRegistry(serviceRegistry);
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestConsumers.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestConsumers.java
index b5c009c..0aead31 100644
--- a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestConsumers.java
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestConsumers.java
@@ -71,23 +71,6 @@ public class TestConsumers extends TestRegistryBase {
     MicroserviceInstanceChangedEvent event = new MicroserviceInstanceChangedEvent();
     event.setKey(key);
 
-    // not match
-    key.setAppId(appId + "1");
-    key.setServiceName(serviceName + "1");
-    eventBus.post(event);
-    Assert.assertEquals(1, microserviceManager.getVersionsByName().size());
-
-    key.setAppId(appId + "1");
-    key.setServiceName(serviceName);
-    eventBus.post(event);
-    Assert.assertEquals(1, microserviceManager.getVersionsByName().size());
-
-    key.setAppId(appId);
-    key.setServiceName(serviceName + "1");
-    eventBus.post(event);
-    Assert.assertEquals(1, microserviceManager.getVersionsByName().size());
-
-    // match
     key.setAppId(appId);
     key.setServiceName(serviceName);
     eventBus.post(event);
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistry.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistry.java
index 63d1d8b..af713bc 100644
--- a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistry.java
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistry.java
@@ -22,7 +22,7 @@ import static org.apache.servicecomb.serviceregistry.RegistryUtils.PUBLISH_ADDRE
 import java.net.InetAddress;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -79,6 +79,7 @@ public class TestRegistry {
     inMemoryConfig.clear();
   }
 
+  @SuppressWarnings("deprecation")
   @Test
   public void testDelegate() {
     ServiceRegistry serviceRegistry = ServiceRegistryFactory.createLocal();
@@ -99,7 +100,7 @@ public class TestRegistry {
     Assert.assertEquals(RegistryUtils.getMicroservice().getServiceId(), instanceList.get(0).getServiceId());
 
     instanceList = RegistryUtils.findServiceInstance("default", "notExists", "0.0.1");
-    Assert.assertEquals(null, instanceList);
+    Assert.assertNull(instanceList);
 
     MicroserviceInstances microserviceInstances =
         RegistryUtils.findServiceInstances("default", "default", "0.0.1", "0");
@@ -193,7 +194,7 @@ public class TestRegistry {
     };
 
     Assert.assertEquals("rest://172.0.0.0:8080", RegistryUtils.getPublishAddress("rest", "172.0.0.0:8080"));
-    Assert.assertEquals(null, RegistryUtils.getPublishAddress("rest", null));
+    Assert.assertNull(RegistryUtils.getPublishAddress("rest", null));
 
     URI uri = new URI(RegistryUtils.getPublishAddress("rest", "0.0.0.0:8080"));
     Assert.assertEquals("1.1.1.1:8080", uri.getAuthority());
@@ -224,7 +225,7 @@ public class TestRegistry {
         };
       }
     };
-    String query = URLEncodedUtils.format(Arrays.asList(new BasicNameValuePair("country", "中 国")),
+    String query = URLEncodedUtils.format(Collections.singletonList(new BasicNameValuePair("country", "中 国")),
         StandardCharsets.UTF_8.name());
     Assert.assertEquals("rest://1.1.1.1:8080?" + query,
         RegistryUtils.getPublishAddress("rest", "172.0.0.0:8080?" + query));
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistryBase.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistryBase.java
index 267af9c..88f63e2 100644
--- a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistryBase.java
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/TestRegistryBase.java
@@ -22,7 +22,6 @@ import org.apache.servicecomb.foundation.test.scaffolding.config.ArchaiusUtils;
 import org.apache.servicecomb.serviceregistry.client.http.MicroserviceInstances;
 import org.apache.servicecomb.serviceregistry.consumer.AppManager;
 import org.apache.servicecomb.serviceregistry.consumer.MicroserviceManager;
-import org.apache.servicecomb.serviceregistry.definition.DefinitionConst;
 import org.apache.servicecomb.serviceregistry.registry.ServiceRegistryFactory;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -30,7 +29,8 @@ import org.junit.Before;
 import com.google.common.eventbus.EventBus;
 
 import mockit.Deencapsulation;
-import mockit.Expectations;
+import mockit.Mock;
+import mockit.MockUp;
 
 public class TestRegistryBase {
   protected ServiceRegistry serviceRegistry;
@@ -88,21 +88,21 @@ public class TestRegistryBase {
   protected void mockNotExist() {
     MicroserviceInstances microserviceInstances = new MicroserviceInstances();
     microserviceInstances.setMicroserviceNotExist(true);
-    new Expectations(RegistryUtils.getServiceRegistry()) {
-      {
-        RegistryUtils.getServiceRegistry()
-            .findServiceInstances(anyString, anyString, DefinitionConst.VERSION_RULE_ALL, anyString);
-        result = microserviceInstances;
+    new MockUp<RegistryUtils>() {
+      @Mock
+      MicroserviceInstances findServiceInstances(String appId, String serviceName,
+          String versionRule, String revision) {
+        return microserviceInstances;
       }
     };
   }
 
   protected void mockDisconnect() {
-    new Expectations(RegistryUtils.getServiceRegistry()) {
-      {
-        RegistryUtils.getServiceRegistry()
-            .findServiceInstances(anyString, anyString, DefinitionConst.VERSION_RULE_ALL, anyString);
-        result = null;
+    new MockUp<RegistryUtils>() {
+      @Mock
+      MicroserviceInstances findServiceInstances(String appId, String serviceName,
+          String versionRule, String revision) {
+        return null;
       }
     };
   }
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/cache/TestMicroserviceInstanceCache.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/cache/TestMicroserviceInstanceCache.java
index 85da7fb..0e31341 100644
--- a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/cache/TestMicroserviceInstanceCache.java
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/cache/TestMicroserviceInstanceCache.java
@@ -18,6 +18,7 @@
 package org.apache.servicecomb.serviceregistry.cache;
 
 import org.apache.servicecomb.serviceregistry.RegistryUtils;
+import org.apache.servicecomb.serviceregistry.ServiceRegistry;
 import org.apache.servicecomb.serviceregistry.api.registry.Microservice;
 import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
 import org.apache.servicecomb.serviceregistry.client.ServiceRegistryClient;
@@ -25,20 +26,25 @@ import org.junit.Assert;
 import org.junit.Test;
 
 import mockit.Expectations;
+import mockit.Mock;
+import mockit.MockUp;
 import mockit.Mocked;
 
 public class TestMicroserviceInstanceCache {
   @Test
-  public void testGetOrCreateMicroservice(@Mocked RegistryUtils utils, @Mocked ServiceRegistryClient client,
+  public void testGetOrCreateMicroservice(@Mocked ServiceRegistry serviceRegistry,
+      @Mocked ServiceRegistryClient client,
       @Mocked Microservice microservice) {
-    new Expectations() {
-      {
-        RegistryUtils.getServiceRegistryClient();
-        result = client;
-        client.getAggregatedMicroservice("forkedid");
-        result = microservice;
-        client.getAggregatedMicroservice("forkedidNull");
-        result = null;
+    new MockUp<RegistryUtils>() {
+      @Mock
+      Microservice getAggregatedRemoteMicroservice(String microserviceId) {
+        if ("forkedid".equals(microserviceId)) {
+          return microservice;
+        }
+        if ("forkedidNull".equals(microserviceId)) {
+          return null;
+        }
+        throw new IllegalArgumentException("unrecognized param");
       }
     };
     Microservice cachedService = MicroserviceInstanceCache.getOrCreate("forkedid");
@@ -50,11 +56,18 @@ public class TestMicroserviceInstanceCache {
   }
 
   @Test
-  public void testGetOrCreateMicroserviceInstance(@Mocked RegistryUtils utils, @Mocked ServiceRegistryClient client,
+  public void testGetOrCreateMicroserviceInstance(@Mocked ServiceRegistry serviceRegistry,
+      @Mocked ServiceRegistryClient client,
       @Mocked MicroserviceInstance instance) {
+    new MockUp<RegistryUtils>() {
+      @Mock
+      ServiceRegistry getServiceRegistry() {
+        return serviceRegistry;
+      }
+    };
     new Expectations() {
       {
-        RegistryUtils.getServiceRegistryClient();
+        serviceRegistry.getServiceRegistryClient();
         result = client;
         client.findServiceInstance("forkedserviceid", "forkedinstanceid");
         result = instance;
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/TestInstanceCacheChecker.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/TestInstanceCacheChecker.java
index 133535c..cafae79 100644
--- a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/TestInstanceCacheChecker.java
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/TestInstanceCacheChecker.java
@@ -177,7 +177,7 @@ public class TestInstanceCacheChecker {
   public void check_findInstances_revisionNotMatch() {
     Holder<MicroserviceInstances> findHolder = createFindServiceInstancesResult();
 
-    new MockUp<ServiceRegistry>(serviceRegistry) {
+    new MockUp<RegistryUtils>() {
       @Mock
       MicroserviceInstances findServiceInstances(String appId, String serviceName,
           String versionRule, String revision) {
diff --git a/spring-boot/spring-boot-common/src/main/java/org/apache/servicecomb/springboot/common/AbstractDiscoveryClient.java b/spring-boot/spring-boot-common/src/main/java/org/apache/servicecomb/springboot/common/AbstractDiscoveryClient.java
index f114439..77b0daa 100644
--- a/spring-boot/spring-boot-common/src/main/java/org/apache/servicecomb/springboot/common/AbstractDiscoveryClient.java
+++ b/spring-boot/spring-boot-common/src/main/java/org/apache/servicecomb/springboot/common/AbstractDiscoveryClient.java
@@ -17,14 +17,15 @@
 package org.apache.servicecomb.springboot.common;
 
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.servicecomb.foundation.common.cache.VersionedCache;
 import org.apache.servicecomb.foundation.common.concurrent.ConcurrentHashMapEx;
 import org.apache.servicecomb.serviceregistry.RegistryUtils;
 import org.apache.servicecomb.serviceregistry.api.registry.Microservice;
-import org.apache.servicecomb.serviceregistry.client.ServiceRegistryClient;
 import org.apache.servicecomb.serviceregistry.definition.DefinitionConst;
 import org.apache.servicecomb.serviceregistry.discovery.DiscoveryContext;
 import org.apache.servicecomb.serviceregistry.discovery.DiscoveryFilter;
@@ -33,9 +34,10 @@ import org.apache.servicecomb.serviceregistry.discovery.DiscoveryTree;
 public abstract class AbstractDiscoveryClient {
 
   private Map<String, DiscoveryTree> discoveryTrees = new ConcurrentHashMapEx<>();
-  private DiscoveryFilter filter = null;
 
-  public AbstractDiscoveryClient(DiscoveryFilter filter){
+  private DiscoveryFilter filter;
+
+  public AbstractDiscoveryClient(DiscoveryFilter filter) {
     this.filter = filter;
   }
 
@@ -44,7 +46,7 @@ public abstract class AbstractDiscoveryClient {
     context.setInputParameters(serviceId);
 
     DiscoveryTree discoveryTree = discoveryTrees.computeIfAbsent(serviceId, key -> {
-      DiscoveryTree tree =  new DiscoveryTree();
+      DiscoveryTree tree = new DiscoveryTree();
       tree.addFilter(filter);
       return tree;
     });
@@ -58,14 +60,14 @@ public abstract class AbstractDiscoveryClient {
   }
 
   public List<String> getServices() {
-    ServiceRegistryClient client = RegistryUtils.getServiceRegistryClient();
-    List<Microservice> services = client.getAllMicroservices();
-    List<String> serviceIDList = new ArrayList<>();
-    if (null != services && !services.isEmpty()) {
-      for (Microservice service : services) {
-        serviceIDList.add(service.getServiceName());
+    Set<String> uniqueServiceNames = new LinkedHashSet<>();
+    RegistryUtils.executeOnEachServiceRegistry(sr -> {
+      List<Microservice> allMicroservices = sr.getServiceRegistryClient().getAllMicroservices();
+      if (null == allMicroservices || allMicroservices.isEmpty()) {
+        return;
       }
-    }
-    return serviceIDList;
+      allMicroservices.forEach(ms -> uniqueServiceNames.add(ms.getServiceName()));
+    });
+    return new ArrayList<>(uniqueServiceNames);
   }
 }