You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@servicecomb.apache.org by GitBox <gi...@apache.org> on 2018/01/02 02:23:08 UTC

[GitHub] yhs0092 closed pull request #475: [JAV-598] Fix invocation failure count in session stickiness rule

yhs0092 closed pull request #475: [JAV-598] Fix invocation failure count in session stickiness rule
URL: https://github.com/apache/incubator-servicecomb-java-chassis/pull/475
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/foundations/foundation-common/src/main/java/io/servicecomb/foundation/common/utils/RollingWindowBucketedCounter.java b/foundations/foundation-common/src/main/java/io/servicecomb/foundation/common/utils/RollingWindowBucketedCounter.java
new file mode 100644
index 000000000..91f00c224
--- /dev/null
+++ b/foundations/foundation-common/src/main/java/io/servicecomb/foundation/common/utils/RollingWindowBucketedCounter.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.servicecomb.foundation.common.utils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * A thread-safe time-windowed rolling counter.
+ * <p>
+ *   In consideration of performance, records are stored in {@link #buckets}, and the {@link #workingBucket} will not be
+ *   counted in the {@link #totalCount}.<br/>
+ *   i.e. If {@link #increment()} is invoked, it will be counted in the {@link #workingBucket}, but this result won't be
+ *   reflected in {@link #totalCount} instantly. <br/>
+ *   When {@link #bucketSizeMs} milliseconds passes, the cursor is shifted, and the value in the {@link #workingBucket}
+ *   will be copied into {@link #buckets}, the oldest value will be overwritten. Then the {@link #workingBucket} will be
+ *   reset to zero. All of the value in {@link #buckets} will be added into {@link #totalCount}.
+ * </p>
+ */
+public class RollingWindowBucketedCounter {
+  /**
+   * how much time a bucket holds, in millisecond
+   */
+  private long bucketSizeMs;
+
+  /**
+   * how many buckets will be counted
+   */
+  private int bucketNum;
+
+  private long[] buckets;
+
+  private AtomicLong workingBucket;
+
+  /**
+   * index of the next written bucket
+   */
+  private int cursor;
+
+  private long shiftTime;
+
+  private long totalCount;
+
+  public RollingWindowBucketedCounter(long bucketSizeMs, int bucketNum) {
+    this.bucketSizeMs = bucketSizeMs;
+    this.bucketNum = bucketNum;
+    this.shiftTime = currentTimeMillis() + bucketSizeMs;
+    this.buckets = new long[this.bucketNum];
+    this.workingBucket = new AtomicLong();
+  }
+
+  public long getTotalCount() {
+    checkAndShift();
+    return totalCount;
+  }
+
+  /**
+   * return the last {@code lastCount} buckets as a list in descending order by time.
+   * <p>
+   *   the first element in the list is the most recent bucket
+   * </p>
+   * @param lastCount how many buckets to list
+   * @return a list of buckets
+   */
+  public List<Long> getLastBuckets(int lastCount) {
+    checkAndShift();
+
+    if (lastCount > bucketNum) {
+      lastCount = bucketNum;
+    }
+
+    List<Long> lastBuckets = new ArrayList<>(lastCount);
+    int offset = cursor - 1;
+    for (int i = 0; i < lastCount; ++i) {
+      if (offset < 0) {
+        offset = buckets.length - 1;
+      }
+
+      lastBuckets.add(buckets[offset]);
+      --offset;
+    }
+
+    return lastBuckets;
+  }
+
+  public long getCurrentBucket() {
+    return this.workingBucket.get();
+  }
+
+  public void increment() {
+    checkAndShift();
+    workingBucket.incrementAndGet();
+  }
+
+  /**
+   * check if it's time to shift bucket, do shift if necessary.
+   */
+  private void checkAndShift() {
+    if (shiftTime <= currentTimeMillis()) {
+      shift();
+    }
+  }
+
+  long currentTimeMillis() {
+    return System.currentTimeMillis();
+  }
+
+  private synchronized void shift() {
+    long current = currentTimeMillis();
+    if (shiftTime > current) {
+      // has been shifted
+      return;
+    }
+
+    long step = (current - shiftTime) / bucketSizeMs + 1;
+    incrementShiftTime(step);
+    buckets[cursor] = workingBucket.getAndSet(0L);
+    shiftCursor();
+
+    // if this counter is not invoked for more than bucketSizeMs milliseconds, additional cursor shift is needed.
+    if (step > 1) {
+      --step;
+      if (step > buckets.length) {
+        // to avoid repeated operation
+        step = buckets.length;
+      }
+      for (int i = 0; i < step; ++i) {
+        buckets[cursor] = 0L;
+        shiftCursor();
+      }
+    }
+
+    refreshTotalCount();
+  }
+
+  private void incrementShiftTime(long step) {
+    shiftTime += bucketSizeMs * step;
+  }
+
+  private void shiftCursor() {
+    ++cursor;
+    if (cursor >= buckets.length) {
+      cursor = 0;
+    }
+  }
+
+  void refreshTotalCount() {
+    long count = 0;
+    for (int i = 0; i < buckets.length; ++i) {
+      count += buckets[i];
+    }
+
+    totalCount = count;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb = new StringBuilder("RollingWindowBucketedCounter{");
+    sb.append("bucketSizeMs=").append(bucketSizeMs);
+    sb.append(", bucketNum=").append(bucketNum);
+    sb.append(", buckets=").append(Arrays.toString(buckets));
+    sb.append(", workingBucket=").append(workingBucket);
+    sb.append(", cursor=").append(cursor);
+    sb.append(", shiftTime=").append(shiftTime);
+    sb.append(", totalCount=").append(totalCount);
+    sb.append('}');
+    return sb.toString();
+  }
+}
diff --git a/foundations/foundation-common/src/test/java/io/servicecomb/foundation/common/utils/RollingWindowBucketedCounterTest.java b/foundations/foundation-common/src/test/java/io/servicecomb/foundation/common/utils/RollingWindowBucketedCounterTest.java
new file mode 100644
index 000000000..4773d6d35
--- /dev/null
+++ b/foundations/foundation-common/src/test/java/io/servicecomb/foundation/common/utils/RollingWindowBucketedCounterTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.servicecomb.foundation.common.utils;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Test;
+
+import mockit.Deencapsulation;
+
+public class RollingWindowBucketedCounterTest {
+
+  private static final long UNIT_TIME = 100L;
+
+  @Test
+  public void testInit() {
+    RollingWindowBucketedCounter counter = new RollingWindowBucketedCounter(UNIT_TIME, 5);
+
+    long[] buckets = Deencapsulation.getField(counter, "buckets");
+    assertEquals(5, buckets.length);
+    for (long bucket : buckets) {
+      assertEquals(0L, bucket);
+    }
+  }
+
+  @Test
+  public void testGetTotalCount() {
+    TestCounter counter = new TestCounter(UNIT_TIME, 5);
+
+    counter.increment();
+    // current increment will be counted in next UNIT_TIME
+    assertEquals(0, counter.getTotalCount());
+
+    for (int i = 0; i < 3; ++i) {
+      counter.addTimeMillis(UNIT_TIME);
+      counter.increment();
+      assertEquals(i + 1, counter.getTotalCount());
+    }
+  }
+
+  @Test
+  public void testGetLastBuckets() {
+    TestCounter counter = new TestCounter(UNIT_TIME, 5);
+
+    for (int i = 0; i < 2; ++i) {
+      counter.increment();
+      counter.addTimeMillis(UNIT_TIME);
+    }
+    counter.addTimeMillis(UNIT_TIME * 2);
+
+    List<Long> bucketList = counter.getLastBuckets(4);
+    assertEquals(4, bucketList.size());
+    assertEquals(Long.valueOf(0), bucketList.get(0));
+    assertEquals(Long.valueOf(0), bucketList.get(1));
+    assertEquals(Long.valueOf(1), bucketList.get(2));
+    assertEquals(Long.valueOf(1), bucketList.get(3));
+  }
+
+  /**
+   * CounterTime = {@link RollingWindowBucketedCounter#bucketSizeMs} * {@link RollingWindowBucketedCounter#bucketNum},
+   * that means the longest time the counter can hold.
+   */
+  @Test
+  public void testIncrementOnTimeIntervalLessThanCounterTime() {
+    TestCounter counter = new TestCounter(UNIT_TIME, 5);
+
+    for (int i = 0; i < 6; ++i) {
+      counter.addTimeMillis(UNIT_TIME);
+      counter.increment();
+      assertEquals(i, counter.getTotalCount());
+    }
+
+    counter.addTimeMillis(3 * UNIT_TIME);
+    long totalCount = counter.getTotalCount();
+    assertEquals(3, totalCount);
+
+    for (int i = 0; i < 2; ++i) {
+      counter.increment();
+      counter.addTimeMillis(UNIT_TIME);
+    }
+    assertEquals(3, counter.getTotalCount());
+    assertEquals(0, ((AtomicLong) Deencapsulation.getField(counter, "workingBucket")).get());
+  }
+
+  @Test
+  public void testIncrementOnTimeIntervalEqualsToCounterTime() {
+    TestCounter counter = new TestCounter(UNIT_TIME, 5);
+    for (int i = 0; i < 6; ++i) {
+      counter.addTimeMillis(UNIT_TIME);
+      counter.increment();
+    }
+
+    counter.addTimeMillis(UNIT_TIME * 5);
+
+    assertEquals(1, counter.getTotalCount());
+  }
+
+  @Test
+  public void testIncrementOnTimeIntervalMoreThanCounterTime() {
+    TestCounter counter = new TestCounter(UNIT_TIME, 5);
+
+    for (int i = 0; i < 6; ++i) {
+      counter.addTimeMillis(UNIT_TIME);
+      counter.increment();
+      assertEquals(i, counter.getTotalCount());
+    }
+
+    counter.addTimeMillis(6 * UNIT_TIME);
+    long totalCount = counter.getTotalCount();
+    assertEquals(0, totalCount);
+
+    for (int i = 0; i < 2; ++i) {
+      counter.increment();
+      counter.addTimeMillis(UNIT_TIME);
+    }
+    assertEquals(2, counter.getTotalCount());
+    assertEquals(0, ((AtomicLong) Deencapsulation.getField(counter, "workingBucket")).get());
+  }
+
+  @Test
+  public void concurrentTest() throws Exception {
+    ExecutorService executorService = Executors.newFixedThreadPool(5);
+    RollingWindowBucketedCounter counter = new RollingWindowBucketedCounter(20, 512);
+    CountDownLatch latch = new CountDownLatch(5);
+    final Runnable task = () -> {
+      for (int i = 0; i < 100; ++i) {
+        try {
+          Thread.sleep(1L);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+        counter.increment();
+      }
+      latch.countDown();
+    };
+    for (int i = 0; i < 5; ++i) {
+      executorService.submit(task);
+    }
+
+    latch.await();
+    // to ensure all of the increment will be counted(not in working bucket)
+    Thread.sleep(30L);
+    assertEquals(100 * 5, counter.getTotalCount());
+  }
+
+  /**
+   * If the time is longer than the CounterTime, some of the olde bucket will be overwritten.
+   */
+  @Test
+  public void concurrentTestOnTimeExceedsCounterTime() throws Exception {
+    ExecutorService executorService = Executors.newFixedThreadPool(5);
+    ConcurrentTestCounter counter = new ConcurrentTestCounter(10, 8);
+    CountDownLatch latch = new CountDownLatch(5);
+    final Runnable task = new Runnable() {
+      @Override
+      public void run() {
+        for (int i = 0; i < 100; ++i) {
+          sleep(1L);
+          counter.increment();
+        }
+        latch.countDown();
+      }
+
+      private void sleep(long millis) {
+        try {
+          Thread.sleep(millis);
+        } catch (InterruptedException e) {
+          e.printStackTrace();
+        }
+      }
+    };
+    for (int i = 0; i < 5; ++i) {
+      executorService.submit(task);
+    }
+
+    latch.await();
+    Thread.sleep(30L);
+    counter.getTotalCount();
+  }
+
+  /**
+   * For test purpose, to avoid the uncertainty of {@link Thread#sleep(long)}
+   */
+  private static class TestCounter extends RollingWindowBucketedCounter {
+    private long timeMillis;
+
+    TestCounter(long bucketSizeMs, int bucketNum) {
+      super(bucketSizeMs, bucketNum);
+    }
+
+    void addTimeMillis(long millis) {
+      this.timeMillis += millis;
+    }
+
+    @Override
+    long currentTimeMillis() {
+      return timeMillis;
+    }
+  }
+
+  /**
+   * For concurrent test purpose
+   */
+  private static class ConcurrentTestCounter extends RollingWindowBucketedCounter {
+    private int bucketNum;
+
+    private List<List<Long>> bucketsSnapshots = new ArrayList<>();
+
+    ConcurrentTestCounter(long bucketSizeMs, int bucketNum) {
+      super(bucketSizeMs, bucketNum);
+      this.bucketNum = bucketNum;
+    }
+
+    @Override
+    protected void refreshTotalCount() {
+      bucketsSnapshots.add(super.getLastBuckets(bucketNum + 2));
+      super.refreshTotalCount();
+      checkState();
+    }
+
+    void checkState() {
+      long result = this.getTotalCount();
+      List<Long> lastBuckets = this.getLastBuckets(bucketNum);
+      long expected = 0;
+      for (Long bucket : lastBuckets) {
+        expected += bucket;
+      }
+
+      assertEquals(expected, result);
+    }
+  }
+}
diff --git a/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/CseServer.java b/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/CseServer.java
index 4f30deb74..515de5a6d 100644
--- a/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/CseServer.java
+++ b/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/CseServer.java
@@ -23,6 +23,7 @@
 
 import io.servicecomb.core.Endpoint;
 import io.servicecomb.core.Transport;
+import io.servicecomb.foundation.common.utils.RollingWindowBucketedCounter;
 import io.servicecomb.serviceregistry.api.registry.MicroserviceInstance;
 import io.servicecomb.serviceregistry.cache.CacheEndpoint;
 
@@ -32,6 +33,11 @@
  *
  */
 public class CseServer extends Server {
+
+  public static final long SAMPLE_UNIT_TIME = 1000L;
+
+  public static final int COUNTER_SIZE = 10;
+
   private final Endpoint endpoint;
 
   // ??????
@@ -44,6 +50,18 @@
    */
   private AtomicInteger continuousFailureCount = new AtomicInteger(0);
 
+  /**
+   * Count the total invocations in the last 10 sec.
+   */
+  private RollingWindowBucketedCounter totalInvocationCount =
+      new RollingWindowBucketedCounter(SAMPLE_UNIT_TIME, COUNTER_SIZE);
+
+  /**
+   * Count the failed invocations in the last 10 sec.
+   */
+  private RollingWindowBucketedCounter failureInvocationCount =
+      new RollingWindowBucketedCounter(SAMPLE_UNIT_TIME, COUNTER_SIZE);
+
   public long getLastVisitTime() {
     return lastVisitTime;
   }
@@ -79,11 +97,11 @@ public String getHost() {
     return endpoint.getEndpoint();
   }
 
-  public void clearContinuousFailure() {
+  void clearContinuousFailure() {
     continuousFailureCount.set(0);
   }
 
-  public void incrementContinuousFailureCount() {
+  void incrementContinuousFailureCount() {
     if (continuousFailureCount.get() < Integer.MAX_VALUE) {
       continuousFailureCount.incrementAndGet();
     }
@@ -93,6 +111,41 @@ public int getCountinuousFailureCount() {
     return continuousFailureCount.get();
   }
 
+  /**
+   * This method should be invoked if an invocation succeeds.
+   */
+  public void invocationSucceeded() {
+    totalInvocationCount.increment();
+    clearContinuousFailure();
+  }
+
+  /**
+   * This method should be invoked if an invocation failed.
+   */
+  public void invocationFailed() {
+    totalInvocationCount.increment();
+    failureInvocationCount.increment();
+    incrementContinuousFailureCount();
+  }
+
+  /**
+   * Get the total invocation count.
+   *
+   * @see #totalInvocationCount
+   */
+  public long getTotalInvocationCount() {
+    return totalInvocationCount.getTotalCount();
+  }
+
+  /**
+   * Get the failed invocation count.
+   *
+   * @see #failureInvocationCount
+   */
+  public long getFailureInvocationCount() {
+    return failureInvocationCount.getTotalCount();
+  }
+
   public boolean equals(Object o) {
     if (o instanceof CseServer) {
       return this.getHost().equals(((CseServer) o).getHost());
diff --git a/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/LoadbalanceHandler.java b/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/LoadbalanceHandler.java
index 61fd9ae68..ea4f79f30 100644
--- a/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/LoadbalanceHandler.java
+++ b/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/LoadbalanceHandler.java
@@ -164,10 +164,10 @@ private void send(Invocation invocation, AsyncResponse asyncResp, final LoadBala
       chosenLB.getLoadBalancerStats().noteResponseTime(server, (System.currentTimeMillis() - time));
       if (resp.isFailed()) {
         chosenLB.getLoadBalancerStats().incrementSuccessiveConnectionFailureCount(server);
-        server.incrementContinuousFailureCount();
+        server.invocationFailed();
       } else {
         chosenLB.getLoadBalancerStats().incrementActiveRequestsCount(server);
-        server.clearContinuousFailure();
+        server.invocationSucceeded();
       }
       asyncResp.handle(resp);
     });
@@ -265,11 +265,11 @@ public void onExecutionFailed(ExecutionContext<Invocation> context, Throwable fi
                     ((Throwable) resp.getResult()).getMessage(),
                     s);
                 chosenLB.getLoadBalancerStats().incrementSuccessiveConnectionFailureCount(s);
-                ((CseServer) s).incrementContinuousFailureCount();
+                ((CseServer) s).invocationFailed();
                 f.onError(resp.getResult());
               } else {
                 chosenLB.getLoadBalancerStats().incrementActiveRequestsCount(s);
-                ((CseServer) s).clearContinuousFailure();
+                ((CseServer) s).invocationSucceeded();
                 chosenLB.getLoadBalancerStats().noteResponseTime(s,
                     (System.currentTimeMillis() - time));
                 f.onNext(resp);
diff --git a/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/SessionStickinessRule.java b/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/SessionStickinessRule.java
index aa2212cbe..483444105 100644
--- a/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/SessionStickinessRule.java
+++ b/handlers/handler-loadbalance/src/main/java/io/servicecomb/loadbalance/SessionStickinessRule.java
@@ -23,10 +23,8 @@
 import com.netflix.loadbalancer.AbstractLoadBalancer;
 import com.netflix.loadbalancer.ILoadBalancer;
 import com.netflix.loadbalancer.IRule;
-import com.netflix.loadbalancer.LoadBalancerStats;
 import com.netflix.loadbalancer.RoundRobinRule;
 import com.netflix.loadbalancer.Server;
-import com.netflix.loadbalancer.ServerStats;
 
 /**
  * ??????????????????????????????????????
@@ -99,15 +97,10 @@ private boolean isTimeOut() {
   }
 
   private boolean isErrorThresholdMet() {
-    AbstractLoadBalancer lb = (AbstractLoadBalancer) getLoadBalancer();
-    LoadBalancerStats stats = lb.getLoadBalancerStats();
-
-    if (stats != null && stats.getServerStats() != null && stats.getServerStats().size() > 0) {
-      ServerStats serverStats = stats.getSingleServerStat(lastServer);
-      int successiveFaildCount = serverStats.getSuccessiveConnectionFailureCount();
+    if (null != lastServer) {
+      long failureInvocationCount = ((CseServer) lastServer).getFailureInvocationCount();
       if (Configuration.INSTANCE.getSuccessiveFailedTimes() > 0
-          && successiveFaildCount >= Configuration.INSTANCE.getSuccessiveFailedTimes()) {
-        serverStats.clearSuccessiveConnectionFailureCount();
+          && failureInvocationCount >= Configuration.INSTANCE.getSuccessiveFailedTimes()) {
         return true;
       }
     }
diff --git a/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestCseServer.java b/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestCseServer.java
index abcd4178a..4c4d05923 100644
--- a/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestCseServer.java
+++ b/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestCseServer.java
@@ -25,7 +25,9 @@
 import org.mockito.Mockito;
 
 import io.servicecomb.core.Transport;
+import io.servicecomb.foundation.common.utils.RollingWindowBucketedCounter;
 import io.servicecomb.serviceregistry.cache.CacheEndpoint;
+import mockit.Deencapsulation;
 
 /**
  *
@@ -91,4 +93,36 @@ public void testClearContinuousFailure() {
     cs.clearContinuousFailure();
     assertEquals(0, cs.getCountinuousFailureCount());
   }
+
+  @Test
+  public void testInvocationSucceeded() {
+    RollingWindowBucketedCounter totalCounter = Mockito.mock(RollingWindowBucketedCounter.class);
+    cs.incrementContinuousFailureCount();
+
+    Deencapsulation.setField(cs, "totalInvocationCount", totalCounter);
+
+    cs.invocationSucceeded();
+
+    Mockito.verify(totalCounter).increment();
+    assertEquals(0, cs.getCountinuousFailureCount());
+  }
+
+  @Test
+  public void testInvocationFailed() {
+    RollingWindowBucketedCounter totalCounter = Mockito.mock(RollingWindowBucketedCounter.class);
+    RollingWindowBucketedCounter failureCounter = Mockito.mock(RollingWindowBucketedCounter.class);
+
+    Deencapsulation.setField(cs, "totalInvocationCount", totalCounter);
+    Deencapsulation.setField(cs, "failureInvocationCount", failureCounter);
+
+    Mockito.when(totalCounter.getTotalCount()).thenReturn(3L);
+    Mockito.when(failureCounter.getTotalCount()).thenReturn(1L);
+    cs.invocationFailed();
+
+    Mockito.verify(totalCounter).increment();
+    Mockito.verify(failureCounter).increment();
+    assertEquals(1, cs.getCountinuousFailureCount());
+    assertEquals(3, cs.getTotalInvocationCount());
+    assertEquals(1, cs.getFailureInvocationCount());
+  }
 }
diff --git a/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestLoadbalanceHandler.java b/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestLoadbalanceHandler.java
index c7e2f0ac5..703ccf72f 100644
--- a/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestLoadbalanceHandler.java
+++ b/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestLoadbalanceHandler.java
@@ -437,16 +437,16 @@ public void sendWithRetry() {
 
     // no exception
   }
-  
+
   @Test
   public void testIsEqual(){
     boolean nullResult = handler.isEqual(null, null);
-    Assert.assertEquals(true, nullResult);  
+    Assert.assertEquals(true, nullResult);
     boolean bothNotNullResult = handler.isEqual("com.netflix.loadbalancer.RandomRule", "com.netflix.loadbalancer.RandomRule");
-    Assert.assertEquals(true, bothNotNullResult);  
+    Assert.assertEquals(true, bothNotNullResult);
     boolean globalNotNull = handler.isEqual(null, "com.netflix.loadbalancer.RandomRule");
-    Assert.assertEquals(false, globalNotNull);  
+    Assert.assertEquals(false, globalNotNull);
     boolean localNotNull = handler.isEqual("com.netflix.loadbalancer.RandomRule", null);
-    Assert.assertEquals(false, localNotNull); 
+    Assert.assertEquals(false, localNotNull);
   }
 }
diff --git a/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestSessionSticknessRule.java b/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestSessionSticknessRule.java
index b75e41175..79948df95 100644
--- a/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestSessionSticknessRule.java
+++ b/handlers/handler-loadbalance/src/test/java/io/servicecomb/loadbalance/TestSessionSticknessRule.java
@@ -17,6 +17,7 @@
 
 package io.servicecomb.loadbalance;
 
+import static org.junit.Assert.assertEquals;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -230,4 +231,33 @@ public void testServerWithKey() {
     }
     Assert.assertTrue(status);
   }
+
+  @Test
+  public void testIsErrorThresholdMet() {
+    SessionStickinessRule ss = new SessionStickinessRule();
+    CseServer server = Mockito.mock(CseServer.class);
+
+    Deencapsulation.setField(ss, "lastServer", server);
+    Mockito.when(server.getFailureInvocationCount()).thenReturn(0L);
+
+    assertEquals(false, Deencapsulation.invoke(ss, "isErrorThresholdMet"));
+  }
+
+  @Test
+  public void testIsErrorThresholdMetOnThresholdIsReached() {
+    SessionStickinessRule ss = new SessionStickinessRule();
+    CseServer server = Mockito.mock(CseServer.class);
+
+    Deencapsulation.setField(ss, "lastServer", server);
+    Mockito.when(server.getFailureInvocationCount()).thenReturn(5L);
+
+    assertEquals(true, Deencapsulation.invoke(ss, "isErrorThresholdMet"));
+  }
+
+  @Test
+  public void testIsErrorThresholdMetOnLastServerIsNull() {
+    SessionStickinessRule ss = new SessionStickinessRule();
+
+    assertEquals(false, Deencapsulation.invoke(ss, "isErrorThresholdMet"));
+  }
 }


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services