You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by li...@apache.org on 2019/08/08 02:15:30 UTC
[servicecomb-java-chassis] 01/02: [SCB-1308] improve instance
isolation recovery mechanism
This is an automated email from the ASF dual-hosted git repository.
liubao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-java-chassis.git
commit 58e8da000eeb3295df16d688a925033ae50a4dad
Author: yhs0092 <yh...@163.com>
AuthorDate: Sun Jun 2 18:29:37 2019 +0800
[SCB-1308] improve instance isolation recovery mechanism
---
.../servicecomb/loadbalance/LoadBalancer.java | 13 +-
.../loadbalance/LoadbalanceHandler.java | 13 +-
.../servicecomb/loadbalance/ServiceCombServer.java | 9 +
.../loadbalance/ServiceCombServerStats.java | 36 ++-
.../filter/IsolationDiscoveryFilter.java | 14 +-
.../loadbalance/TestLoadBalanceHandler2.java | 137 ++++++++++++
.../filter/IsolationDiscoveryFilterTest.java | 245 +++++++++++++++++++++
7 files changed, 458 insertions(+), 9 deletions(-)
diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadBalancer.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadBalancer.java
index 4843fc9..6439cec 100644
--- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadBalancer.java
+++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadBalancer.java
@@ -28,7 +28,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.netflix.loadbalancer.LoadBalancerStats;
-
/**
* A load balancer with RuleExt and ServerListFilterExt
*/
@@ -65,7 +64,17 @@ public class LoadBalancer {
break;
}
}
- return rule.choose(servers, invocation);
+ ServiceCombServer server = rule.choose(servers, invocation);
+ if (null == server) {
+ return null;
+ }
+ ServiceCombServerStats serverStats = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(server);
+ if (serverStats.isIsolated()) {
+ LOGGER.info("The Service {}'s instance {} has been isolated for a while, give a single test opportunity.",
+ invocation.getMicroserviceName(),
+ server.getInstance().getInstanceId());
+ }
+ return server;
}
public LoadBalancerStats getLoadBalancerStats() {
diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadbalanceHandler.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadbalanceHandler.java
index 7fd82c1..ae4456e 100644
--- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadbalanceHandler.java
+++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/LoadbalanceHandler.java
@@ -40,6 +40,7 @@ import org.apache.servicecomb.core.provider.consumer.SyncResponseExecutor;
import org.apache.servicecomb.foundation.common.cache.VersionedCache;
import org.apache.servicecomb.foundation.common.concurrent.ConcurrentHashMapEx;
import org.apache.servicecomb.foundation.common.utils.ExceptionUtils;
+import org.apache.servicecomb.loadbalance.filter.IsolationDiscoveryFilter;
import org.apache.servicecomb.loadbalance.filter.ServerDiscoveryFilter;
import org.apache.servicecomb.serviceregistry.discovery.DiscoveryContext;
import org.apache.servicecomb.serviceregistry.discovery.DiscoveryFilter;
@@ -94,6 +95,7 @@ public class LoadbalanceHandler implements Handler {
@Override
public Server chooseServer(Object key) {
+ boolean isRetry = null != lastServer;
for (int i = 0; i < COUNT; i++) {
Server s = delegate.chooseServer((Invocation) key);
if (s != null && !s.equals(lastServer)) {
@@ -101,11 +103,13 @@ public class LoadbalanceHandler implements Handler {
break;
}
}
+ if (isRetry) {
+ LOGGER.info("retry to instance [{}]", lastServer.getHostPort());
+ }
return lastServer;
}
-
@Override
public void markServerDown(Server server) {
throw new UnsupportedOperationException("Not implemented.");
@@ -184,6 +188,13 @@ public class LoadbalanceHandler implements Handler {
@Override
public void handle(Invocation invocation, AsyncResponse asyncResp) throws Exception {
+ AsyncResponse response = asyncResp;
+ asyncResp = async -> {
+ if (Boolean.TRUE.equals(invocation.getLocalContext(IsolationDiscoveryFilter.TRYING_INSTANCES_EXISTING))) {
+ ServiceCombServerStats.releaseTryingChance();
+ }
+ response.handle(async);
+ };
if (supportDefinedEndpoint) {
if (defineEndpointAndHandle(invocation, asyncResp)) {
return;
diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServer.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServer.java
index 800e2ae..e780b8b 100644
--- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServer.java
+++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServer.java
@@ -17,6 +17,9 @@
package org.apache.servicecomb.loadbalance;
+import java.net.URI;
+import java.net.URISyntaxException;
+
import org.apache.servicecomb.core.Endpoint;
import org.apache.servicecomb.core.Transport;
import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
@@ -64,6 +67,12 @@ public class ServiceCombServer extends Server {
// To make all rules work only on "how to choose a server from alive servers", we do not rely on Robbin defined status
this.setAlive(true);
this.setReadyToServe(true);
+ try {
+ URI endpointURI = new URI(endpoint.getEndpoint());
+ setHost(endpointURI.getHost());
+ setPort(endpointURI.getPort());
+ } catch (URISyntaxException ignored) {
+ }
}
public Endpoint getEndpoint() {
diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServerStats.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServerStats.java
index b33771b..b41e2da 100644
--- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServerStats.java
+++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/ServiceCombServerStats.java
@@ -17,8 +17,12 @@
package org.apache.servicecomb.loadbalance;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Special stats that com.netflix.loadbalancer.ServerStats not provided.
*
@@ -27,9 +31,19 @@ import java.util.concurrent.atomic.AtomicLong;
public class ServiceCombServerStats {
private static final long TIME_WINDOW_IN_MILLISECONDS = 60000;
+ private static final Logger LOGGER = LoggerFactory.getLogger(ServiceCombServerStats.class);
+
+ /**
+ * There is not more than 1 server allowed to stay in TRYING status concurrently.
+ * If the value of globalAllowIsolatedServerTryingFlag is false, there have been 1 server in
+ * TRYING status, so the other isolated servers cannot be transferred into
+ * TRYING status; otherwise the value of globalAllowIsolatedServerTryingFlag is true.
+ */
+ private static AtomicBoolean globalAllowIsolatedServerTryingFlag = new AtomicBoolean(true);
+
private long lastWindow = System.currentTimeMillis();
- private Object lock = new Object();
+ private final Object lock = new Object();
private AtomicLong continuousFailureCount = new AtomicLong(0);
@@ -45,6 +59,23 @@ public class ServiceCombServerStats {
private boolean isolated = false;
+ public static boolean isolatedServerCanTry() {
+ return globalAllowIsolatedServerTryingFlag.get();
+ }
+
+ /**
+ * Applying for a trying chance for the isolated server. There is only 1 trying chance globally concurrently.
+ *
+ * @return true if the chance is applied successfully, otherwise false
+ */
+ public static boolean applyForTryingChance() {
+ return isolatedServerCanTry() && globalAllowIsolatedServerTryingFlag.compareAndSet(true, false);
+ }
+
+ public static void releaseTryingChance() {
+ globalAllowIsolatedServerTryingFlag.set(true);
+ }
+
public void markIsolated(boolean isolated) {
this.isolated = isolated;
}
@@ -57,6 +88,9 @@ public class ServiceCombServerStats {
totalRequests.incrementAndGet();
successRequests.incrementAndGet();
continuousFailureCount.set(0);
+ if (isolated) {
+ LOGGER.info("trying server invocation success!");
+ }
}
public void markFailure() {
diff --git a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilter.java b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilter.java
index 4328d73..256a6c0 100644
--- a/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilter.java
+++ b/handlers/handler-loadbalance/src/main/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilter.java
@@ -44,6 +44,8 @@ import com.netflix.config.DynamicPropertyFactory;
*/
public class IsolationDiscoveryFilter implements DiscoveryFilter {
+ public static final String TRYING_INSTANCES_EXISTING = "scb-hasTryingInstances";
+
private static final Logger LOGGER = LoggerFactory.getLogger(IsolationDiscoveryFilter.class);
private static final String EMPTY_INSTANCE_PROTECTION = "servicecomb.loadbalance.filter.isolation.emptyInstanceProtectionEnabled";
@@ -142,14 +144,16 @@ public class IsolationDiscoveryFilter implements DiscoveryFilter {
if (!checkThresholdAllowed(settings, serverStats)) {
if (serverStats.isIsolated()
&& (System.currentTimeMillis() - serverStats.getLastVisitTime()) > settings.singleTestTime) {
+ if (!ServiceCombServerStats.applyForTryingChance()) {
+ // this server hasn't been isolated for enough long time, or there is no trying chance
+ return false;
+ }
// [1]we can implement better recovery based on several attempts, but here we do not know if this attempt is success
- LOGGER.info("The Service {}'s instance {} has been isolated for a while, give a single test opportunity.",
- invocation.getMicroserviceName(),
- instance.getInstanceId());
+ invocation.addLocalContext(TRYING_INSTANCES_EXISTING, Boolean.TRUE);
return true;
}
if (!serverStats.isIsolated()) {
- ServiceCombLoadBalancerStats.INSTANCE.markIsolated(server, true);
+ serverStats.markIsolated(true);
eventBus.post(
new IsolationServerEvent(invocation, instance, serverStats,
settings, Type.OPEN, server.getEndpoint()));
@@ -164,7 +168,7 @@ public class IsolationDiscoveryFilter implements DiscoveryFilter {
if ((System.currentTimeMillis() - serverStats.getLastVisitTime()) <= settings.minIsolationTime) {
return false;
}
- ServiceCombLoadBalancerStats.INSTANCE.markIsolated(server, false);
+ serverStats.markIsolated(false);
eventBus.post(new IsolationServerEvent(invocation, instance, serverStats,
settings, Type.CLOSE, server.getEndpoint()));
LOGGER.warn("Recover service {}'s instance {} from isolation.", invocation.getMicroserviceName(),
diff --git a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java
index f0e3ad5..e3d08d1 100644
--- a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java
+++ b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/TestLoadBalanceHandler2.java
@@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit;
import javax.xml.ws.Holder;
import org.apache.servicecomb.core.CseContext;
+import org.apache.servicecomb.core.Endpoint;
import org.apache.servicecomb.core.Invocation;
import org.apache.servicecomb.core.NonSwaggerInvocation;
import org.apache.servicecomb.core.SCBEngine;
@@ -53,6 +54,7 @@ import org.apache.servicecomb.serviceregistry.cache.InstanceCacheManager;
import org.apache.servicecomb.serviceregistry.discovery.DiscoveryTree;
import org.apache.servicecomb.serviceregistry.discovery.DiscoveryTreeNode;
import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -79,12 +81,14 @@ public class TestLoadBalanceHandler2 {
public void setUp() {
// clear up load balance stats
ServiceCombLoadBalancerStats.INSTANCE.init();
+ ServiceCombServerStats.releaseTryingChance();
}
@After
public void teardown() {
CseContext.getInstance().setTransportManager(null);
ArchaiusUtils.resetConfig();
+ ServiceCombServerStats.releaseTryingChance();
}
@Test
@@ -821,4 +825,137 @@ public class TestLoadBalanceHandler2 {
Assert.assertTrue(e.getMessage().contains("the endpoint's transport is not found."));
}
}
+
+ @Test
+ public void trying_chance_should_be_released() {
+ List<ServiceCombServer> servers = new ArrayList<>();
+ ServiceCombServer serviceCombServer = createMockedServer("instanceId", "rest://127.0.0.1:8080");
+ servers.add(serviceCombServer);
+
+ DiscoveryTree discoveryTree = createMockedDiscoveryTree(servers);
+ LoadbalanceHandler handler = new LoadbalanceHandler(discoveryTree);
+
+ // mock the process of the isolated server selected and changed to TRYING status
+ ServiceCombServerStats serviceCombServerStats =
+ mockServiceCombServerStats(serviceCombServer, 5, true);
+
+ Invocation invocation = new NonSwaggerInvocation("testApp", "testMicroserviceName", "0.0.0+",
+ (inv, aysnc) -> {
+ Assert.assertEquals("rest://127.0.0.1:8080", inv.getEndpoint().getEndpoint());
+ Assert.assertTrue(serviceCombServerStats.isIsolated());
+ Assert.assertEquals(5, serviceCombServerStats.getCountinuousFailureCount());
+ Assert.assertFalse(ServiceCombServerStats.isolatedServerCanTry());
+ aysnc.success("OK");
+ });
+
+ Assert.assertTrue(ServiceCombServerStats.applyForTryingChance());
+ invocation.addLocalContext(IsolationDiscoveryFilter.TRYING_INSTANCES_EXISTING, true);
+ try {
+ handler.handle(invocation, (response) -> Assert.assertEquals("OK", response.getResult()));
+ } catch (Exception e) {
+ Assert.fail("unexpected exception " + e.getMessage());
+ }
+ Assert.assertEquals("rest://127.0.0.1:8080", invocation.getEndpoint().getEndpoint());
+ Assert.assertTrue(serviceCombServerStats.isIsolated());
+ Assert.assertEquals(0, serviceCombServerStats.getCountinuousFailureCount());
+ Assert.assertTrue(ServiceCombServerStats.isolatedServerCanTry());
+ }
+
+ /**
+ * Two available instances, first time the normal instance is selected and failed. Then retry to the TRYING status
+ * instance. In the whole procedure, the TRYING status instance should keep the TRYING status.
+ */
+ @Test
+ public void first_normal_instance_then_trying_instance() {
+ ExtensionsManager.addExtentionsFactory(new DefaultRetryExtensionsFactory());
+ ArchaiusUtils.setProperty("servicecomb.loadbalance.retryEnabled", true);
+ ArchaiusUtils.setProperty("servicecomb.loadbalance.retryOnNext", 1);
+
+ ArrayList<ServiceCombServer> servers = new ArrayList<>();
+ ServiceCombServer server0 = createMockedServer("instanceId0", "rest://127.0.0.1:8080");
+ ServiceCombServer server1 = createMockedServer("instanceId1", "rest://127.0.0.1:8081");
+ servers.add(server0);
+ servers.add(server1);
+
+ ServiceCombServerStats stats0 = mockServiceCombServerStats(server0, 0, false);
+ ServiceCombServerStats stats1 = mockServiceCombServerStats(server1, 5, true);
+
+ DiscoveryTree discoveryTree = createMockedDiscoveryTree(servers);
+ LoadbalanceHandler handler = new LoadbalanceHandler(discoveryTree);
+
+ Holder<Integer> counter = new Holder<>(0);
+ Invocation invocation = new NonSwaggerInvocation("testApp", "testMicroserviceName", "0.0.0+",
+ (inv, aysnc) -> {
+ Assert.assertFalse(stats0.isIsolated());
+ Assert.assertTrue(stats1.isIsolated());
+ Assert.assertEquals(5, stats1.getCountinuousFailureCount());
+ Assert.assertFalse(ServiceCombServerStats.isolatedServerCanTry());
+ if (counter.value == 0) {
+ Assert.assertEquals("rest://127.0.0.1:8080", inv.getEndpoint().getEndpoint());
+ Assert.assertEquals(0, stats0.getCountinuousFailureCount());
+ counter.value++;
+ aysnc.producerFail(new InvocationException(503, "RETRY", "retry to next instance"));
+ } else if (counter.value == 1) {
+ Assert.assertEquals("rest://127.0.0.1:8081", inv.getEndpoint().getEndpoint());
+ Assert.assertEquals(1, stats0.getCountinuousFailureCount());
+ counter.value++;
+ aysnc.success("OK");
+ } else {
+ aysnc.producerFail(new InvocationException(400, "UNEXPECTED", "Unexpected Counter Value"));
+ }
+ });
+
+ Assert.assertTrue(ServiceCombServerStats.applyForTryingChance());
+ invocation.addLocalContext(IsolationDiscoveryFilter.TRYING_INSTANCES_EXISTING, true);
+ try {
+ handler.handle(invocation, (response) -> Assert.assertEquals("OK", response.getResult()));
+ } catch (Exception e) {
+ Assert.fail("unexpected exception " + e.getMessage());
+ }
+ Assert.assertEquals("rest://127.0.0.1:8081", invocation.getEndpoint().getEndpoint());
+ Assert.assertFalse(stats0.isIsolated());
+ Assert.assertEquals(1, stats0.getCountinuousFailureCount());
+ Assert.assertTrue(stats1.isIsolated());
+ Assert.assertEquals(0, stats1.getCountinuousFailureCount());
+ Assert.assertTrue(ServiceCombServerStats.isolatedServerCanTry());
+ }
+
+ /**
+ * Mock the statistics of the specified {@code serviceCombServer}, set the failureCount and status.
+ * @return the ServiceCombServerStats object corresponding to the param {@code serviceCombServer}
+ */
+ private ServiceCombServerStats mockServiceCombServerStats(ServiceCombServer serviceCombServer, int failureCount,
+ boolean isIsolatedStatus) {
+ ServiceCombServerStats serviceCombServerStats =
+ ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(serviceCombServer);
+ for (int i = 0; i < failureCount; ++i) {
+ serviceCombServerStats.markFailure();
+ }
+ serviceCombServerStats.markIsolated(isIsolatedStatus);
+ return serviceCombServerStats;
+ }
+
+ /**
+ * Create a mocked ServiceCombServer with specified microserviceInstanceId and endpoint.
+ */
+ private ServiceCombServer createMockedServer(String microserviceInstanceId, String endpoint) {
+ MicroserviceInstance microserviceInstance = new MicroserviceInstance();
+ microserviceInstance.setInstanceId(microserviceInstanceId);
+ return new ServiceCombServer(
+ new Endpoint(Mockito.mock(Transport.class), endpoint),
+ microserviceInstance);
+ }
+
+ /**
+ * Create a mocked DiscoveryTree that always returns {@code servers} as the versionedCache result.
+ */
+ private DiscoveryTree createMockedDiscoveryTree(List<ServiceCombServer> servers) {
+ DiscoveryTree discoveryTree = Mockito.mock(DiscoveryTree.class);
+ DiscoveryTreeNode versionedCache = new DiscoveryTreeNode()
+ .name("testVersionedCacheName")
+ .data(servers);
+ Mockito.when(discoveryTree.discovery(Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
+ .thenReturn(versionedCache);
+ return discoveryTree;
+ }
}
diff --git a/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilterTest.java b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilterTest.java
new file mode 100644
index 0000000..a632352
--- /dev/null
+++ b/handlers/handler-loadbalance/src/test/java/org/apache/servicecomb/loadbalance/filter/IsolationDiscoveryFilterTest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.loadbalance.filter;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.core.Transport;
+import org.apache.servicecomb.loadbalance.Configuration;
+import org.apache.servicecomb.loadbalance.ServiceCombLoadBalancerStats;
+import org.apache.servicecomb.loadbalance.ServiceCombServer;
+import org.apache.servicecomb.loadbalance.ServiceCombServerStats;
+import org.apache.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
+import org.apache.servicecomb.serviceregistry.cache.CacheEndpoint;
+import org.apache.servicecomb.serviceregistry.discovery.DiscoveryContext;
+import org.apache.servicecomb.serviceregistry.discovery.DiscoveryTreeNode;
+import org.hamcrest.Matchers;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import mockit.Deencapsulation;
+import mockit.Mocked;
+
+public class IsolationDiscoveryFilterTest {
+
+ private DiscoveryContext discoveryContext;
+
+ private DiscoveryTreeNode discoveryTreeNode;
+
+ private Map<String, MicroserviceInstance> data;
+
+ private IsolationDiscoveryFilter filter;
+
+ @Mocked
+ private Transport transport = Mockito.mock(Transport.class);
+
+ private Invocation invocation = new Invocation() {
+ @Override
+ public String getMicroserviceName() {
+ return "testMicroserviceName";
+ }
+ };
+
+ @Before
+ public void before() {
+ discoveryContext = new DiscoveryContext();
+ discoveryContext.setInputParameters(invocation);
+ discoveryTreeNode = new DiscoveryTreeNode();
+ Mockito.doAnswer(a -> a.getArguments()[0]).when(transport).parseAddress(Mockito.anyString());
+ data = new HashMap<>();
+ for (int i = 0; i < 3; ++i) {
+ MicroserviceInstance instance = new MicroserviceInstance();
+ instance.setInstanceId("i" + i);
+ String endpoint = "rest://127.0.0.1:" + i;
+ instance.setEndpoints(Collections.singletonList(endpoint));
+ data.put(instance.getInstanceId(), instance);
+ ServiceCombServer serviceCombServer = new ServiceCombServer(transport, new CacheEndpoint(endpoint, instance));
+ ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(serviceCombServer);
+ }
+ discoveryTreeNode.data(data);
+
+ filter = new IsolationDiscoveryFilter();
+ ServiceCombServerStats.releaseTryingChance();
+ }
+
+ @After
+ public void after() {
+ Deencapsulation.invoke(ServiceCombLoadBalancerStats.INSTANCE, "init");
+ ServiceCombServerStats.releaseTryingChance();
+ }
+
+ @Test
+ public void discovery_no_instance_reach_error_threshold() {
+ DiscoveryTreeNode childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+
+ Map<String, MicroserviceInstance> childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i0", "i1", "i2"));
+ Assert.assertEquals(data.get("i0"), childNodeData.get("i0"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+ }
+
+ @Test
+ public void discovery_isolate_error_instance() {
+ ServiceCombServer server0 = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServer(data.get("i0"));
+ for (int i = 0; i < 4; ++i) {
+ ServiceCombLoadBalancerStats.INSTANCE.markFailure(server0);
+ }
+ DiscoveryTreeNode childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ Map<String, MicroserviceInstance> childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i0", "i1", "i2"));
+ Assert.assertEquals(data.get("i0"), childNodeData.get("i0"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+
+ // by default 5 times continuous failure will cause isolation
+ ServiceCombLoadBalancerStats.INSTANCE.markFailure(server0);
+ Assert.assertFalse(ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(server0).isIsolated());
+
+ childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i1", "i2"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+ Assert.assertTrue(ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(server0).isIsolated());
+ }
+
+ @Test
+ public void discovery_try_isolated_instance_after_singleTestTime() {
+ ServiceCombServer server0 = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServer(data.get("i0"));
+ ServiceCombServerStats serviceCombServerStats = ServiceCombLoadBalancerStats.INSTANCE
+ .getServiceCombServerStats(server0);
+ for (int i = 0; i < 5; ++i) {
+ serviceCombServerStats.markFailure();
+ }
+ letIsolatedInstancePassSingleTestTime(serviceCombServerStats);
+ ServiceCombLoadBalancerStats.INSTANCE.markIsolated(server0, true);
+
+ Assert.assertTrue(ServiceCombServerStats.isolatedServerCanTry());
+ Assert.assertFalse(
+ Boolean.TRUE.equals(invocation.getLocalContext(IsolationDiscoveryFilter.TRYING_INSTANCES_EXISTING)));
+ DiscoveryTreeNode childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ Map<String, MicroserviceInstance> childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i0", "i1", "i2"));
+ Assert.assertEquals(data.get("i0"), childNodeData.get("i0"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+ Assert.assertTrue(serviceCombServerStats.isIsolated());
+ Assert.assertFalse(ServiceCombServerStats.isolatedServerCanTry());
+ Assert.assertTrue(
+ Boolean.TRUE.equals(invocation.getLocalContext(IsolationDiscoveryFilter.TRYING_INSTANCES_EXISTING)));
+ }
+
+ @Test
+ public void discovery_not_try_isolated_instance_concurrently() {
+ ServiceCombServer server0 = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServer(data.get("i0"));
+ ServiceCombServerStats serviceCombServerStats = ServiceCombLoadBalancerStats.INSTANCE
+ .getServiceCombServerStats(server0);
+ for (int i = 0; i < 5; ++i) {
+ serviceCombServerStats.markFailure();
+ }
+ ServiceCombLoadBalancerStats.INSTANCE.markIsolated(server0, true);
+ letIsolatedInstancePassSingleTestTime(serviceCombServerStats);
+
+ Assert.assertTrue(ServiceCombServerStats.isolatedServerCanTry());
+
+ // The first invocation can occupy the trying chance
+ DiscoveryTreeNode childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ Map<String, MicroserviceInstance> childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i0", "i1", "i2"));
+ Assert.assertEquals(data.get("i0"), childNodeData.get("i0"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+ Assert.assertFalse(ServiceCombServerStats.isolatedServerCanTry());
+
+ // Other invocation cannot get trying chance concurrently
+ childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i1", "i2"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+
+ ServiceCombServerStats.releaseTryingChance(); // after the first invocation releases the trying chance
+
+ // Other invocation can get the trying chance
+ childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i0", "i1", "i2"));
+ Assert.assertEquals(data.get("i0"), childNodeData.get("i0"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+ Assert.assertFalse(ServiceCombServerStats.isolatedServerCanTry());
+ }
+
+ private ServiceCombServerStats letIsolatedInstancePassSingleTestTime(ServiceCombServerStats serviceCombServerStats) {
+ Deencapsulation.setField(serviceCombServerStats, "lastActiveTime",
+ System.currentTimeMillis() - 1 - Configuration.INSTANCE.getSingleTestTime(invocation.getMicroserviceName()));
+ Deencapsulation.setField(serviceCombServerStats, "lastVisitTime",
+ System.currentTimeMillis() - 1 - Configuration.INSTANCE.getSingleTestTime(invocation.getMicroserviceName()));
+ return serviceCombServerStats;
+ }
+
+ @Test
+ public void discovery_keep_minIsolationTime() {
+ ServiceCombServer server0 = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServer(data.get("i0"));
+ ServiceCombLoadBalancerStats.INSTANCE.markIsolated(server0, true);
+ ServiceCombLoadBalancerStats.INSTANCE.markSuccess(server0);
+
+ DiscoveryTreeNode childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ Map<String, MicroserviceInstance> childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i1", "i2"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+
+ ServiceCombServerStats serviceCombServerStats = ServiceCombLoadBalancerStats.INSTANCE
+ .getServiceCombServerStats(server0);
+ Deencapsulation.setField(serviceCombServerStats, "lastVisitTime",
+ System.currentTimeMillis() - Configuration.INSTANCE.getMinIsolationTime(invocation.getMicroserviceName()) - 1);
+ childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i0", "i1", "i2"));
+ Assert.assertEquals(data.get("i0"), childNodeData.get("i0"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+ }
+
+ @Test
+ public void discovery_recover_instance() {
+ ServiceCombServer server0 = ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServer(data.get("i0"));
+ ServiceCombLoadBalancerStats.INSTANCE.markSuccess(server0);
+ ServiceCombServerStats serviceCombServerStats = ServiceCombLoadBalancerStats.INSTANCE
+ .getServiceCombServerStats(server0);
+ Deencapsulation.setField(serviceCombServerStats, "lastVisitTime",
+ System.currentTimeMillis() - Configuration.INSTANCE.getMinIsolationTime(invocation.getMicroserviceName()) - 1);
+
+ ServiceCombLoadBalancerStats.INSTANCE.markIsolated(server0, true);
+ DiscoveryTreeNode childNode = filter.discovery(discoveryContext, discoveryTreeNode);
+ Map<String, MicroserviceInstance> childNodeData = childNode.data();
+ Assert.assertThat(childNodeData.keySet(), Matchers.containsInAnyOrder("i0", "i1", "i2"));
+ Assert.assertEquals(data.get("i0"), childNodeData.get("i0"));
+ Assert.assertEquals(data.get("i1"), childNodeData.get("i1"));
+ Assert.assertEquals(data.get("i2"), childNodeData.get("i2"));
+ Assert.assertFalse(ServiceCombLoadBalancerStats.INSTANCE.getServiceCombServerStats(server0).isIsolated());
+ }
+}
\ No newline at end of file