You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by wu...@apache.org on 2018/07/19 06:47:56 UTC

[incubator-servicecomb-java-chassis] 01/05: [SCB-729] check instance cache between sdk and sc

This is an automated email from the ASF dual-hosted git repository.

wujimin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-servicecomb-java-chassis.git

commit 9ebfc055a4eb58b42f1a626d7a5031b45b9abcaf
Author: wujimin <wu...@huawei.com>
AuthorDate: Mon Jul 16 11:08:12 2018 +0800

    [SCB-729] check instance cache between sdk and sc
---
 .../serviceregistry/consumer/AppManager.java       |   4 +
 .../consumer/MicroserviceManager.java              |   4 +
 .../consumer/MicroserviceVersions.java             |  20 +-
 .../serviceregistry/diagnosis/Status.java          |  24 ++
 .../diagnosis/instance/InstanceCacheChecker.java   | 140 ++++++++++++
 .../diagnosis/instance/InstanceCacheResult.java    |  62 ++++++
 .../diagnosis/instance/InstanceCacheSummary.java   |  79 +++++++
 .../consumer/TestMicroserviceVersions.java         |   1 +
 .../instance/TestInstanceCacheChecker.java         | 245 +++++++++++++++++++++
 9 files changed, 578 insertions(+), 1 deletion(-)

diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/AppManager.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/AppManager.java
index 8093ceb..7cc45e2 100644
--- a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/AppManager.java
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/AppManager.java
@@ -43,6 +43,10 @@ public class AppManager {
     return microserviceVersionFactory;
   }
 
+  public Map<String, MicroserviceManager> getApps() {
+    return apps;
+  }
+
   public void setMicroserviceVersionFactory(MicroserviceVersionFactory microserviceVersionFactory) {
     this.microserviceVersionFactory = microserviceVersionFactory;
   }
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceManager.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceManager.java
index a7bd4ae..2d16d0d 100644
--- a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceManager.java
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceManager.java
@@ -45,6 +45,10 @@ public class MicroserviceManager {
     appManager.getEventBus().register(this);
   }
 
+  public Map<String, MicroserviceVersions> getVersionsByName() {
+    return versionsByName;
+  }
+
   public MicroserviceVersions getOrCreateMicroserviceVersions(String microserviceName) {
     MicroserviceVersions microserviceVersions = versionsByName.computeIfAbsent(microserviceName, name -> {
       MicroserviceVersions instance = new MicroserviceVersions(appManager, appId, microserviceName);
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceVersions.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceVersions.java
index 169328e..f867f6a 100644
--- a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceVersions.java
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/consumer/MicroserviceVersions.java
@@ -17,6 +17,7 @@
 
 package org.apache.servicecomb.serviceregistry.consumer;
 
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -48,8 +49,15 @@ public class MicroserviceVersions {
 
   private String microserviceName;
 
+  // revision and pulledInstances directly equals to SC's response
   private String revision = null;
 
+  private List<MicroserviceInstance> pulledInstances;
+
+  // instances not always equals to pulledInstances
+  // in the future:
+  //  pulledInstances means all instance
+  //  instances means available instance
   private List<MicroserviceInstance> instances;
 
   // key is service id
@@ -101,6 +109,14 @@ public class MicroserviceVersions {
     return (T) versions.get(serviceId);
   }
 
+  public String getRevision() {
+    return revision;
+  }
+
+  public List<MicroserviceInstance> getPulledInstances() {
+    return pulledInstances;
+  }
+
   public void submitPull() {
     pendingPullCount.incrementAndGet();
 
@@ -127,7 +143,9 @@ public class MicroserviceVersions {
     if (!microserviceInstances.isNeedRefresh()) {
       return;
     }
-    List<MicroserviceInstance> pulledInstances = microserviceInstances.getInstancesResponse().getInstances();
+
+    pulledInstances = microserviceInstances.getInstancesResponse().getInstances();
+    pulledInstances.sort(Comparator.comparing(MicroserviceInstance::getInstanceId));
     String rev = microserviceInstances.getRevision();
 
     safeSetInstances(pulledInstances, rev);
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/Status.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/Status.java
new file mode 100644
index 0000000..47aedd4
--- /dev/null
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/Status.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.serviceregistry.diagnosis;
+
+public enum Status {
+  NORMAL,
+  ABNORMAL,
+  UNKNOWN
+}
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheChecker.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheChecker.java
new file mode 100644
index 0000000..8e0e1d7
--- /dev/null
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheChecker.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.serviceregistry.diagnosis.instance;
+
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.servicecomb.serviceregistry.RegistryUtils;
+import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
+import org.apache.servicecomb.serviceregistry.client.http.MicroserviceInstances;
+import org.apache.servicecomb.serviceregistry.consumer.AppManager;
+import org.apache.servicecomb.serviceregistry.consumer.MicroserviceManager;
+import org.apache.servicecomb.serviceregistry.consumer.MicroserviceVersions;
+import org.apache.servicecomb.serviceregistry.definition.DefinitionConst;
+import org.apache.servicecomb.serviceregistry.diagnosis.Status;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.vertx.core.json.Json;
+
+public class InstanceCacheChecker {
+  private static final Logger LOGGER = LoggerFactory.getLogger(InstanceCacheChecker.class);
+
+  private AppManager appManager;
+
+  private Set<Status> statuses = new HashSet<>();
+
+  private InstanceCacheSummary instanceCacheSummary = new InstanceCacheSummary();
+
+  public InstanceCacheChecker(AppManager appManager) {
+    this.appManager = appManager;
+  }
+
+  public InstanceCacheSummary check() {
+    instanceCacheSummary.setAppId(RegistryUtils.getMicroservice().getAppId());
+    instanceCacheSummary.setMicroserviceName(RegistryUtils.getMicroservice().getServiceName());
+    instanceCacheSummary.setTimestamp(System.currentTimeMillis());
+
+    for (MicroserviceManager microserviceManager : appManager.getApps().values()) {
+      for (MicroserviceVersions microserviceVersions : microserviceManager.getVersionsByName().values()) {
+        InstanceCacheResult instanceCacheResult = check(microserviceVersions);
+        addInstanceCacheResult(instanceCacheResult);
+      }
+    }
+
+    generateStatus();
+
+    return instanceCacheSummary;
+  }
+
+  private void addInstanceCacheResult(InstanceCacheResult instanceCacheResult) {
+    statuses.add(instanceCacheResult.getStatus());
+    instanceCacheSummary.getProducers().add(instanceCacheResult);
+  }
+
+  protected InstanceCacheResult check(MicroserviceVersions microserviceVersions) {
+    InstanceCacheResult instanceCacheResult = new InstanceCacheResult();
+    instanceCacheResult.setAppId(microserviceVersions.getAppId());
+    instanceCacheResult.setMicroserviceName(microserviceVersions.getMicroserviceName());
+
+    MicroserviceInstances microserviceInstances = RegistryUtils
+        .findServiceInstances(microserviceVersions.getAppId(),
+            microserviceVersions.getMicroserviceName(),
+            DefinitionConst.VERSION_RULE_ALL,
+            null);
+    if (microserviceInstances == null) {
+      instanceCacheResult.setStatus(Status.UNKNOWN);
+      instanceCacheResult.setDetail("failed to find instances from service center");
+      return instanceCacheResult;
+    }
+    if (microserviceInstances.isMicroserviceNotExist()) {
+      // no problem, will be deleted from MicroserviceManager in next pull
+      instanceCacheResult.setStatus(Status.UNKNOWN);
+      instanceCacheResult.setDetail("microservice is not exist anymore, will be deleted from memory in next pull");
+      return instanceCacheResult;
+    }
+
+    if (!Objects.equals(microserviceInstances.getRevision(), microserviceVersions.getRevision())) {
+      // maybe not pull, wait for next pull we get the same revision
+      instanceCacheResult.setStatus(Status.UNKNOWN);
+      instanceCacheResult.setDetail(String.format(
+          "revision is different, will be synchronized in next pull. local revision=%s, remote revision=%s",
+          microserviceVersions.getRevision(), microserviceInstances.getRevision()));
+      return instanceCacheResult;
+    }
+
+    // compare all instances
+    List<MicroserviceInstance> remoteInstances = microserviceInstances.getInstancesResponse().getInstances();
+//    if (RandomUtils.nextInt(0, 2) == 0) {
+//      MicroserviceInstance microserviceInstance = new MicroserviceInstance();
+//      microserviceInstance.setInstanceId("abc");
+//      remoteInstances.add(microserviceInstance);
+//    }
+    remoteInstances.sort(Comparator.comparing(MicroserviceInstance::getInstanceId));
+    String local = Json.encode(microserviceVersions.getPulledInstances());
+    String remote = Json.encode(remoteInstances);
+    if (local.equals(remote)) {
+      instanceCacheResult.setStatus(Status.NORMAL);
+      return instanceCacheResult;
+    }
+
+    LOGGER.error("instance cache not match. appId={}, microservice={}.\n"
+            + "local cache: {}\n"
+            + "remote cache: {}",
+        microserviceVersions.getAppId(),
+        microserviceVersions.getMicroserviceName(),
+        local,
+        remote);
+    instanceCacheResult.setStatus(Status.ABNORMAL);
+    instanceCacheResult.setDetail("instance cache not match");
+    return instanceCacheResult;
+  }
+
+  protected void generateStatus() {
+    if (statuses.contains(Status.ABNORMAL)) {
+      instanceCacheSummary.setStatus(Status.ABNORMAL);
+    } else if (statuses.contains(Status.UNKNOWN)) {
+      instanceCacheSummary.setStatus(Status.UNKNOWN);
+    } else {
+      instanceCacheSummary.setStatus(Status.NORMAL);
+    }
+  }
+}
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheResult.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheResult.java
new file mode 100644
index 0000000..a5ae3de
--- /dev/null
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheResult.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.serviceregistry.diagnosis.instance;
+
+import org.apache.servicecomb.serviceregistry.diagnosis.Status;
+
+public class InstanceCacheResult {
+  // producer appId and microserviceName
+  private String appId;
+
+  private String microserviceName;
+
+  private Status status;
+
+  private String detail;
+
+  public String getAppId() {
+    return appId;
+  }
+
+  public void setAppId(String appId) {
+    this.appId = appId;
+  }
+
+  public String getMicroserviceName() {
+    return microserviceName;
+  }
+
+  public void setMicroserviceName(String microserviceName) {
+    this.microserviceName = microserviceName;
+  }
+
+  public Status getStatus() {
+    return status;
+  }
+
+  public void setStatus(Status status) {
+    this.status = status;
+  }
+
+  public String getDetail() {
+    return detail;
+  }
+
+  public void setDetail(String detail) {
+    this.detail = detail;
+  }
+}
diff --git a/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheSummary.java b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheSummary.java
new file mode 100644
index 0000000..9a02cf8
--- /dev/null
+++ b/service-registry/src/main/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/InstanceCacheSummary.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.serviceregistry.diagnosis.instance;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.servicecomb.serviceregistry.diagnosis.Status;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+
+public class InstanceCacheSummary {
+  // consumer appId and microserviceName
+  @JsonIgnore
+  private String appId;
+
+  @JsonIgnore
+  private String microserviceName;
+
+  private Status status;
+
+  private List<InstanceCacheResult> producers = new ArrayList<>();
+
+  private long timestamp;
+
+  public String getAppId() {
+    return appId;
+  }
+
+  public void setAppId(String appId) {
+    this.appId = appId;
+  }
+
+  public String getMicroserviceName() {
+    return microserviceName;
+  }
+
+  public void setMicroserviceName(String microserviceName) {
+    this.microserviceName = microserviceName;
+  }
+
+  public Status getStatus() {
+    return status;
+  }
+
+  public void setStatus(Status status) {
+    this.status = status;
+  }
+
+  public long getTimestamp() {
+    return timestamp;
+  }
+
+  public void setTimestamp(long timestamp) {
+    this.timestamp = timestamp;
+  }
+
+  public List<InstanceCacheResult> getProducers() {
+    return producers;
+  }
+
+  public void setProducers(List<InstanceCacheResult> producers) {
+    this.producers = producers;
+  }
+}
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/consumer/TestMicroserviceVersions.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/consumer/TestMicroserviceVersions.java
index 8bd0a2d..9614048 100644
--- a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/consumer/TestMicroserviceVersions.java
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/consumer/TestMicroserviceVersions.java
@@ -73,6 +73,7 @@ public class TestMicroserviceVersions {
 
   @Before
   public void setUp() throws Exception {
+    ArchaiusUtils.resetConfig();
     microserviceInstances = new MicroserviceInstances();
     findInstancesResponse = new FindInstancesResponse();
   }
diff --git a/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/TestInstanceCacheChecker.java b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/TestInstanceCacheChecker.java
new file mode 100644
index 0000000..d4d55bf
--- /dev/null
+++ b/service-registry/src/test/java/org/apache/servicecomb/serviceregistry/diagnosis/instance/TestInstanceCacheChecker.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.servicecomb.serviceregistry.diagnosis.instance;
+
+import java.util.ArrayList;
+
+import javax.xml.ws.Holder;
+
+import org.apache.servicecomb.serviceregistry.RegistryUtils;
+import org.apache.servicecomb.serviceregistry.ServiceRegistry;
+import org.apache.servicecomb.serviceregistry.api.registry.Microservice;
+import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
+import org.apache.servicecomb.serviceregistry.api.response.FindInstancesResponse;
+import org.apache.servicecomb.serviceregistry.client.http.MicroserviceInstances;
+import org.apache.servicecomb.serviceregistry.definition.DefinitionConst;
+import org.apache.servicecomb.serviceregistry.diagnosis.Status;
+import org.apache.servicecomb.serviceregistry.registry.ServiceRegistryFactory;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import io.vertx.core.json.Json;
+import mockit.Mock;
+import mockit.MockUp;
+
+public class TestInstanceCacheChecker {
+  ServiceRegistry serviceRegistry = ServiceRegistryFactory.createLocal();
+
+  InstanceCacheChecker checker;
+
+  InstanceCacheSummary expectedSummary = new InstanceCacheSummary();
+
+  String appId = "appId";
+
+  String microserviceName = "msName";
+
+  @Before
+  public void setUp() throws Exception {
+    serviceRegistry.init();
+    RegistryUtils.setServiceRegistry(serviceRegistry);
+
+    checker = new InstanceCacheChecker(serviceRegistry.getAppManager());
+    expectedSummary.setStatus(Status.NORMAL);
+    expectedSummary.setTimestamp(1);
+
+    new MockUp<System>() {
+      @Mock
+      long currentTimeMillis() {
+        return 1L;
+      }
+    };
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    RegistryUtils.setServiceRegistry(null);
+  }
+
+  @Test
+  public void check_appManager_empty() {
+    InstanceCacheSummary instanceCacheSummary = checker.check();
+
+    Assert.assertEquals(Json.encode(expectedSummary), Json.encode(instanceCacheSummary));
+  }
+
+  @Test
+  public void check_microserviceManager_empty() {
+    appId = "notExist";
+    serviceRegistry.getAppManager().getOrCreateMicroserviceVersions(appId, microserviceName);
+    InstanceCacheSummary instanceCacheSummary = checker.check();
+
+    InstanceCacheResult instanceCacheResult = new InstanceCacheResult();
+    instanceCacheResult.setAppId(appId);
+    instanceCacheResult.setMicroserviceName(microserviceName);
+    instanceCacheResult.setStatus(Status.NORMAL);
+    expectedSummary.getProducers().add(instanceCacheResult);
+
+    Assert.assertEquals(Json.encode(expectedSummary), Json.encode(instanceCacheSummary));
+  }
+
+  protected Holder<MicroserviceInstances> createFindServiceInstancesResult() {
+    MicroserviceInstances microserviceInstances = new MicroserviceInstances();
+    microserviceInstances.setNeedRefresh(true);
+    microserviceInstances.setRevision("first");
+    FindInstancesResponse findInstancesResponse = new FindInstancesResponse();
+    findInstancesResponse.setInstances(new ArrayList<>());
+    microserviceInstances.setInstancesResponse(findInstancesResponse);
+
+    Holder<MicroserviceInstances> findHolder = new Holder<>();
+    findHolder.value = microserviceInstances;
+    return findHolder;
+  }
+
+  protected void registerMicroservice(String appId, String microserviceName) {
+    Microservice microservice = new Microservice();
+    microservice.setAppId(appId);
+    microservice.setServiceName(microserviceName);
+    microservice.setVersion("1.0.0");
+
+    serviceRegistry.getServiceRegistryClient().registerMicroservice(microservice);
+  }
+
+  @Test
+  public void check_findInstances_failed() {
+    Holder<MicroserviceInstances> findHolder = createFindServiceInstancesResult();
+
+    new MockUp<RegistryUtils>() {
+      @Mock
+      MicroserviceInstances findServiceInstances(String appId, String serviceName,
+          String versionRule, String revision) {
+        return findHolder.value;
+      }
+    };
+
+    registerMicroservice(appId, microserviceName);
+
+    serviceRegistry.getAppManager()
+        .getOrCreateMicroserviceVersionRule(appId, microserviceName, DefinitionConst.VERSION_RULE_ALL);
+
+    findHolder.value = null;
+    InstanceCacheSummary instanceCacheSummary = checker.check();
+
+    InstanceCacheResult instanceCacheResult = new InstanceCacheResult();
+    instanceCacheResult.setAppId(appId);
+    instanceCacheResult.setMicroserviceName(microserviceName);
+    instanceCacheResult.setStatus(Status.UNKNOWN);
+    instanceCacheResult.setDetail("failed to find instances from service center");
+    expectedSummary.getProducers().add(instanceCacheResult);
+    expectedSummary.setStatus(Status.UNKNOWN);
+
+    Assert.assertEquals(Json.encode(expectedSummary), Json.encode(instanceCacheSummary));
+  }
+
+  @Test
+  public void check_findInstances_serviceNotExist() {
+    Holder<MicroserviceInstances> findHolder = createFindServiceInstancesResult();
+
+    new MockUp<RegistryUtils>() {
+      @Mock
+      MicroserviceInstances findServiceInstances(String appId, String serviceName,
+          String versionRule, String revision) {
+        return findHolder.value;
+      }
+    };
+
+    registerMicroservice(appId, microserviceName);
+
+    serviceRegistry.getAppManager()
+        .getOrCreateMicroserviceVersionRule(appId, microserviceName, DefinitionConst.VERSION_RULE_ALL);
+
+    findHolder.value.setMicroserviceNotExist(true);
+    InstanceCacheSummary instanceCacheSummary = checker.check();
+
+    InstanceCacheResult instanceCacheResult = new InstanceCacheResult();
+    instanceCacheResult.setAppId(appId);
+    instanceCacheResult.setMicroserviceName(microserviceName);
+    instanceCacheResult.setStatus(Status.UNKNOWN);
+    instanceCacheResult.setDetail("microservice is not exist anymore, will be deleted from memory in next pull");
+    expectedSummary.getProducers().add(instanceCacheResult);
+    expectedSummary.setStatus(Status.UNKNOWN);
+
+    Assert.assertEquals(Json.encode(expectedSummary), Json.encode(instanceCacheSummary));
+  }
+
+  @Test
+  public void check_findInstances_revisionNotMatch() {
+    Holder<MicroserviceInstances> findHolder = createFindServiceInstancesResult();
+
+    new MockUp<RegistryUtils>() {
+      @Mock
+      MicroserviceInstances findServiceInstances(String appId, String serviceName,
+          String versionRule, String revision) {
+        return findHolder.value;
+      }
+    };
+
+    registerMicroservice(appId, microserviceName);
+
+    serviceRegistry.getAppManager()
+        .getOrCreateMicroserviceVersionRule(appId, microserviceName, DefinitionConst.VERSION_RULE_ALL);
+
+    findHolder.value.setRevision("second");
+    InstanceCacheSummary instanceCacheSummary = checker.check();
+
+    InstanceCacheResult instanceCacheResult = new InstanceCacheResult();
+    instanceCacheResult.setAppId(appId);
+    instanceCacheResult.setMicroserviceName(microserviceName);
+    instanceCacheResult.setStatus(Status.UNKNOWN);
+    instanceCacheResult.setDetail(
+        "revision is different, will be synchronized in next pull. local revision=first, remote revision=second");
+    expectedSummary.getProducers().add(instanceCacheResult);
+    expectedSummary.setStatus(Status.UNKNOWN);
+
+    Assert.assertEquals(Json.encode(expectedSummary), Json.encode(instanceCacheSummary));
+  }
+
+  @Test
+  public void check_findInstances_cacheNotMatch() {
+    Holder<MicroserviceInstances> findHolder = createFindServiceInstancesResult();
+
+    new MockUp<RegistryUtils>() {
+      @Mock
+      MicroserviceInstances findServiceInstances(String appId, String serviceName,
+          String versionRule, String revision) {
+        return findHolder.value;
+      }
+    };
+
+    registerMicroservice(appId, microserviceName);
+
+    serviceRegistry.getAppManager()
+        .getOrCreateMicroserviceVersionRule(appId, microserviceName, DefinitionConst.VERSION_RULE_ALL);
+
+    Holder<MicroserviceInstances> newFindHolder = createFindServiceInstancesResult();
+    newFindHolder.value.getInstancesResponse().getInstances().add(new MicroserviceInstance());
+    findHolder.value = newFindHolder.value;
+    InstanceCacheSummary instanceCacheSummary = checker.check();
+
+    InstanceCacheResult instanceCacheResult = new InstanceCacheResult();
+    instanceCacheResult.setAppId(appId);
+    instanceCacheResult.setMicroserviceName(microserviceName);
+    instanceCacheResult.setStatus(Status.ABNORMAL);
+    instanceCacheResult.setDetail(
+        "instance cache not match");
+    expectedSummary.getProducers().add(instanceCacheResult);
+    expectedSummary.setStatus(Status.ABNORMAL);
+
+    Assert.assertEquals(Json.encode(expectedSummary), Json.encode(instanceCacheSummary));
+  }
+}