You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@servicecomb.apache.org by li...@apache.org on 2018/03/22 09:16:37 UTC

[incubator-servicecomb-java-chassis] 02/05: Review comments fix:Fault-Injection handler

This is an automated email from the ASF dual-hosted git repository.

liubao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-servicecomb-java-chassis.git

commit ebd1cedb00d387afda8096778d1098777f8a4783
Author: maheshrajus <ma...@huawei.com>
AuthorDate: Wed Mar 21 18:19:22 2018 +0530

    Review comments fix:Fault-Injection handler
---
 .../servicecomb/faultinjection/AbortFault.java     | 26 ++++----
 .../servicecomb/faultinjection/DelayFault.java     | 54 ++++++++---------
 .../apache/servicecomb/faultinjection/Fault.java   |  3 +-
 .../servicecomb/faultinjection/FaultExecutor.java  | 70 ++++++++++++++++++++++
 .../faultinjection/FaultInjectionConfig.java       |  6 +-
 .../faultinjection/FaultInjectionConst.java        |  8 +--
 .../faultinjection/FaultInjectionHandler.java      | 34 +++++++----
 .../faultinjection/FaultInjectionUtil.java         | 10 ++--
 .../servicecomb/faultinjection/FaultParam.java     | 12 ++++
 .../faultinjection/TestFaultInjectConfig.java      | 11 ++--
 .../faultinjection/TestFaultInjectHandler.java     | 56 +++++++++++------
 11 files changed, 207 insertions(+), 83 deletions(-)

diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbortFault.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbortFault.java
index 3bf4d84..c878899 100644
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbortFault.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbortFault.java
@@ -18,6 +18,7 @@
 package org.apache.servicecomb.faultinjection;
 
 import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
 import org.apache.servicecomb.swagger.invocation.exception.CommonExceptionData;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -28,14 +29,15 @@ public class AbortFault extends AbstractFault {
   private static final Logger LOGGER = LoggerFactory.getLogger(FaultInjectionConfig.class);
 
   @Override
-  public FaultResponse injectFault(Invocation invocation, FaultParam faultParam) {
+  public void injectFault(Invocation invocation, FaultParam faultParam, AsyncResponse asynResponse) {
     // get the config values related to delay.
     int abortPercent = FaultInjectionUtil.getFaultInjectionConfig(invocation,
         "abort.percent");
 
-    if (abortPercent == FaultInjectionConst.FAULT_INJECTION_CFG_NULL) {
-      LOGGER.info("Fault injection: Abort percentage is not configured");
-      return new FaultResponse();
+    if (abortPercent == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+      LOGGER.debug("Fault injection: Abort percentage is not configured");
+      asynResponse.success(new FaultResponse());
+      return;
     }
 
     // check fault abort condition.
@@ -45,20 +47,24 @@ public class AbortFault extends AbstractFault {
       int errorCode = FaultInjectionUtil.getFaultInjectionConfig(invocation,
           "abort.httpStatus");
 
-      if (errorCode == FaultInjectionConst.FAULT_INJECTION_CFG_NULL) {
-        LOGGER.info("Fault injection: Abort error code is not configured");
-        return new FaultResponse();
+      if (errorCode == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+        LOGGER.debug("Fault injection: Abort error code is not configured");
+        asynResponse.success(new FaultResponse());
+        return;
       }
       // if request need to be abort then return failure with given error code
       CommonExceptionData errorData = new CommonExceptionData("aborted by fault inject");
-      return new FaultResponse(-1, errorCode, errorData);
+
+      FaultResponse response = new FaultResponse(FaultInjectionConst.FAULT_INJECTION_ERROR, errorCode, errorData);
+      asynResponse.success(response);
+      return;
     }
 
-    return new FaultResponse();
+    asynResponse.success(new FaultResponse());
   }
 
   @Override
   public int getPriority() {
-    return FaultInjectionConst.FAULTINJECTION_PRIORITY_MIN;
+    return 200;
   }
 }
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/DelayFault.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/DelayFault.java
index 5ab0ed9..aaf2f63 100644
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/DelayFault.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/DelayFault.java
@@ -17,10 +17,8 @@
 
 package org.apache.servicecomb.faultinjection;
 
-import java.util.concurrent.CountDownLatch;
-
 import org.apache.servicecomb.core.Invocation;
-import org.apache.servicecomb.foundation.vertx.VertxUtils;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
@@ -34,48 +32,50 @@ public class DelayFault extends AbstractFault {
 
   @Override
   public int getPriority() {
-    return FaultInjectionConst.FAULTINJECTION_PRIORITY_MAX;
+    return 100;
   }
 
   @Override
-  public FaultResponse injectFault(Invocation invocation, FaultParam faultAttributes) {
+  public void injectFault(Invocation invocation, FaultParam faultParam, AsyncResponse asynResponse) {
     int delayPercent = FaultInjectionUtil.getFaultInjectionConfig(invocation,
         "delay.percent");
 
-    if (delayPercent == FaultInjectionConst.FAULT_INJECTION_CFG_NULL) {
-      LOGGER.info("Fault injection: delay percentage is not configured");
-      return new FaultResponse();
+    if (delayPercent == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+      LOGGER.debug("Fault injection: delay percentage is not configured");
+      asynResponse.success(new FaultResponse());
+      return;
     }
 
     // check fault delay condition.
-    boolean isDelay = FaultInjectionUtil.checkFaultInjectionDelayAndAbort(faultAttributes.getReqCount(), delayPercent);
+    boolean isDelay = FaultInjectionUtil.checkFaultInjectionDelayAndAbort(faultParam.getReqCount(), delayPercent);
     if (isDelay) {
-      LOGGER.info("Fault injection: delay is added for the request by fault inject handler");
+      LOGGER.debug("Fault injection: delay is added for the request by fault inject handler");
       long delay = FaultInjectionUtil.getFaultInjectionConfig(invocation,
           "delay.fixedDelay");
 
-      if (delay == FaultInjectionConst.FAULT_INJECTION_CFG_NULL) {
-        LOGGER.info("Fault injection: delay is not configured");
-        return new FaultResponse();
+      if (delay == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+        LOGGER.debug("Fault injection: delay is not configured");
+        asynResponse.success(new FaultResponse());
+        return;
       }
 
-      CountDownLatch latch = new CountDownLatch(1);
-      Vertx vertx = VertxUtils.getOrCreateVertxByName("faultinjection", null);
-      vertx.setTimer(delay, new Handler<Long>() {
-        @Override
-        public void handle(Long timeID) {
-          latch.countDown();
+      Vertx vertx = faultParam.getVertx();
+      if (vertx != null) {
+        vertx.setTimer(delay, new Handler<Long>() {
+          @Override
+          public void handle(Long timeID) {
+            asynResponse.success(new FaultResponse());
+          }
+        });
+      } else {
+        try {
+          Thread.sleep(delay);
+        } catch (InterruptedException e) {
+          LOGGER.info("Interrupted exception is received");
         }
-      });
-
-      try {
-        latch.await();
-      } catch (InterruptedException e) {
-        LOGGER.info("Interrupted exception is received");
+        asynResponse.success(new FaultResponse());
       }
     }
-
-    return new FaultResponse();
   }
 
 }
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/Fault.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/Fault.java
index 9d17e07..6f50c62 100644
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/Fault.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/Fault.java
@@ -18,10 +18,11 @@
 package org.apache.servicecomb.faultinjection;
 
 import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
 
 public interface Fault {
 
   int getPriority();
 
-  FaultResponse injectFault(Invocation invocation, FaultParam faultAttributes);
+  void injectFault(Invocation invocation, FaultParam faultAttributes, AsyncResponse asynResponse);
 }
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultExecutor.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultExecutor.java
new file mode 100644
index 0000000..291b1c8
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultExecutor.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import java.util.List;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+import org.apache.servicecomb.swagger.invocation.Response;
+
+
+/**
+ * Implements the fault feature execution one after other.
+ */
+public class FaultExecutor {
+  private List<Fault> faultInjectList;
+
+  private int handlerIndex = 0;
+
+  private Invocation invocation;
+
+  private FaultParam param;
+
+  public FaultExecutor(List<Fault> faultInjectList, Invocation invocation, FaultParam param) {
+    this.faultInjectList = faultInjectList;
+    this.invocation = invocation;
+    this.param = param;
+  }
+
+  public void execute(AsyncResponse asyncResponse) {
+    this.next(asyncResponse);
+  }
+
+  private void next(AsyncResponse asyncResponse) {
+    if (handlerIndex >= faultInjectList.size()) {
+      asyncResponse.complete(Response.succResp(new FaultResponse()));
+      return;
+    }
+
+    int runIndex = handlerIndex;
+    handlerIndex++;
+    faultInjectList.get(runIndex).injectFault(invocation, param, response -> {
+      if (response.isFailed()) {
+        asyncResponse.complete(response);
+      } else {
+        FaultResponse r = response.getResult();
+        if (r.getStatusCode() != 0) {
+          asyncResponse.complete(response);
+        } else {
+          FaultExecutor.this.next(asyncResponse);
+        }
+      }
+    });
+  }
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConfig.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConfig.java
index f6000e5..d43eb2f 100755
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConfig.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConfig.java
@@ -42,8 +42,7 @@ public final class FaultInjectionConfig {
     DynamicIntProperty dynamicIntProperty = DynamicPropertyFactory.getInstance().getIntProperty(config,
         defaultValue);
 
-    if (cfgCallback.get(config) == null) {
-      cfgCallback.put(config, config);
+    cfgCallback.computeIfAbsent(config, key -> {
       dynamicIntProperty.addCallback(() -> {
         int newValue = dynamicIntProperty.get();
         String cfgName = dynamicIntProperty.getName();
@@ -52,7 +51,8 @@ public final class FaultInjectionConfig {
         FaultInjectionUtil.setConfigCenterValue(cfgName, new AtomicInteger(newValue));
         LOGGER.info("{} changed to {}", cfgName, newValue);
       });
-    }
+      return config;
+    });
 
     return dynamicIntProperty.get();
   }
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConst.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConst.java
index fdc2a8b..b1221e7 100644
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConst.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConst.java
@@ -22,15 +22,13 @@ package org.apache.servicecomb.faultinjection;
  */
 public class FaultInjectionConst {
 
-  public static final int FAULT_INJECTION_CFG_NULL = -1;
+  public static final int FAULT_INJECTION_DEFAULT_VALUE = -1;
+
+  public static final int FAULT_INJECTION_ERROR = -1;
 
   public static final String CONSUMER_FAULTINJECTION = "cse.governance.Consumer.";
 
   public static final String CONSUMER_FAULTINJECTION_GLOBAL = "cse.governance.Consumer._global.";
 
   public static final String CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS = "policy.fault.protocols.";
-
-  public static final int FAULTINJECTION_PRIORITY_MIN = 10;
-
-  public static final int FAULTINJECTION_PRIORITY_MAX = 1;
 }
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionHandler.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionHandler.java
index 090b5a9..204049c 100755
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionHandler.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionHandler.java
@@ -24,9 +24,11 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.servicecomb.core.Handler;
 import org.apache.servicecomb.core.Invocation;
 import org.apache.servicecomb.swagger.invocation.AsyncResponse;
-import org.apache.servicecomb.swagger.invocation.context.HttpStatus;
 import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
 
+import io.vertx.core.Context;
+import io.vertx.core.Vertx;
+
 /**
  * Fault injection handler which injects the delay/abort for requests based on
  * the percentage configured in service file.
@@ -57,16 +59,28 @@ public class FaultInjectionHandler implements Handler {
     // increment the request count here after checking the delay/abort condition.
     reqCount.incrementAndGet();
 
-    for (Fault fault : faultInjectionFeatureList) {
-      FaultResponse faultResponse = fault.injectFault(invocation, new FaultParam(reqCountCurrent));
-
-      if (faultResponse.getStatusCode() != 0) {
-        asyncResp.consumerFail(new InvocationException(new HttpStatus(faultResponse.getErrorCode(),
-            "invocation exception induced by fault injection"), faultResponse.getErrorData()));
-        return;
-      }
+    FaultParam param = new FaultParam(reqCountCurrent);
+    Context currentContext = Vertx.currentContext();
+    if (currentContext != null && currentContext.isEventLoopContext()) {
+      param.setVertx(currentContext.owner());
     }
 
-    invocation.next(asyncResp);
+    FaultExecutor executor = new FaultExecutor(faultInjectionFeatureList, invocation, param);
+    executor.execute(response -> {
+      try {
+        if (response.isFailed()) {
+          asyncResp.complete(response);
+        } else {
+          FaultResponse r = response.getResult();
+          if (r.getStatusCode() == FaultInjectionConst.FAULT_INJECTION_ERROR) {
+            asyncResp.consumerFail(new InvocationException(r.getErrorCode(), "", r.getErrorData()));
+            return;
+          }
+          invocation.next(asyncResp);
+        }
+      } catch (Exception e) {
+        asyncResp.consumerFail(e);
+      }
+    });
   }
 }
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionUtil.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionUtil.java
index 2ff4abf..18dba50 100755
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionUtil.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionUtil.java
@@ -20,7 +20,7 @@ package org.apache.servicecomb.faultinjection;
 import static org.apache.servicecomb.faultinjection.FaultInjectionConst.CONSUMER_FAULTINJECTION;
 import static org.apache.servicecomb.faultinjection.FaultInjectionConst.CONSUMER_FAULTINJECTION_GLOBAL;
 import static org.apache.servicecomb.faultinjection.FaultInjectionConst.CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS;
-import static org.apache.servicecomb.faultinjection.FaultInjectionConst.FAULT_INJECTION_CFG_NULL;
+import static org.apache.servicecomb.faultinjection.FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE;
 
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -90,7 +90,7 @@ public class FaultInjectionUtil {
         + CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS + invocation.getTransport().getName() + "." + key;
 
     value = getConfigValue(config);
-    if ((value != FAULT_INJECTION_CFG_NULL)) {
+    if ((value != FAULT_INJECTION_DEFAULT_VALUE)) {
       return value;
     }
 
@@ -98,14 +98,14 @@ public class FaultInjectionUtil {
         + CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS + invocation.getTransport().getName() + "." + key;
 
     value = getConfigValue(config);
-    if ((value != FAULT_INJECTION_CFG_NULL)) {
+    if ((value != FAULT_INJECTION_DEFAULT_VALUE)) {
       return value;
     }
 
     config = CONSUMER_FAULTINJECTION + serviceName + "." + CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS
         + invocation.getTransport().getName() + "." + key;
     value = getConfigValue(config);
-    if ((value != FAULT_INJECTION_CFG_NULL)) {
+    if ((value != FAULT_INJECTION_DEFAULT_VALUE)) {
       return value;
     }
 
@@ -130,7 +130,7 @@ public class FaultInjectionUtil {
       return cfgMap.get(config).get();
     }
 
-    value = FaultInjectionConfig.getConfigVal(config, FAULT_INJECTION_CFG_NULL);
+    value = FaultInjectionConfig.getConfigVal(config, FAULT_INJECTION_DEFAULT_VALUE);
     return value;
 
   }
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultParam.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultParam.java
index 0dcd19b..92f0e84 100644
--- a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultParam.java
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultParam.java
@@ -17,12 +17,16 @@
 
 package org.apache.servicecomb.faultinjection;
 
+import io.vertx.core.Vertx;
+
 /**
  * Fault injection parameters which decides the fault injection condition.
  */
 public class FaultParam {
   private long reqCount;
 
+  private Vertx vertx;
+
   public long getReqCount() {
     return reqCount;
   }
@@ -34,4 +38,12 @@ public class FaultParam {
   FaultParam(long reqCount) {
     this.reqCount = reqCount;
   }
+
+  public Vertx getVertx() {
+    return vertx;
+  }
+
+  public void setVertx(Vertx vertx) {
+    this.vertx = vertx;
+  }
 }
diff --git a/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectConfig.java b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectConfig.java
index f4d3137..4c2a08e 100644
--- a/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectConfig.java
+++ b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectConfig.java
@@ -71,16 +71,17 @@ public class TestFaultInjectConfig {
   public void testConstants() {
     assertEquals("cse.governance.Consumer.", FaultInjectionConst.CONSUMER_FAULTINJECTION);
     assertEquals("policy.fault.protocols.", FaultInjectionConst.CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS);
-    assertEquals(-1, FaultInjectionConst.FAULT_INJECTION_CFG_NULL);
+    assertEquals(-1, FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE);
     assertEquals("cse.governance.Consumer._global.", FaultInjectionConst.CONSUMER_FAULTINJECTION_GLOBAL);
-    assertEquals(10, FaultInjectionConst.FAULTINJECTION_PRIORITY_MIN);
-    assertEquals(1, FaultInjectionConst.FAULTINJECTION_PRIORITY_MAX);
+    assertEquals(-1, FaultInjectionConst.FAULT_INJECTION_ERROR);
   }
 
   @Test
   public void testFaultParam() {
     faultParam.setReqCount(100);
+    faultParam.setVertx(null);
     assertEquals(100, faultParam.getReqCount());
+    assertEquals(null, faultParam.getVertx());
   }
 
   @Test
@@ -96,7 +97,7 @@ public class TestFaultInjectConfig {
 
   @Test
   public void testFaultPriority() {
-    assertEquals(10, abortFault.getPriority());
-    assertEquals(1, delayFault.getPriority());
+    assertEquals(200, abortFault.getPriority());
+    assertEquals(100, delayFault.getPriority());
   }
 }
diff --git a/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectHandler.java b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectHandler.java
index 7288f78..600c59a 100644
--- a/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectHandler.java
+++ b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectHandler.java
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import org.apache.servicecomb.core.Invocation;
 import org.apache.servicecomb.core.Transport;
 import org.apache.servicecomb.core.definition.OperationMeta;
+import org.apache.servicecomb.foundation.vertx.VertxUtils;
 import org.apache.servicecomb.swagger.invocation.AsyncResponse;
 import org.apache.servicecomb.swagger.invocation.Response;
 import org.junit.After;
@@ -36,6 +37,8 @@ import org.mockito.InjectMocks;
 import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
 
+import io.vertx.core.Vertx;
+
 /**
  * Tests the fault injection handler functionality.
  */
@@ -139,7 +142,7 @@ public class TestFaultInjectHandler {
     Mockito.when(invocation.getSchemaId()).thenReturn("sayHelloSchema");
     Mockito.when(invocation.getMicroserviceName()).thenReturn("hello");
 
-    List<Fault> faultInjectionFeatureList = Arrays.asList(abortFault, delayFault);
+    List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
     handler.setFaultFeature(faultInjectionFeatureList);
 
     handler.handle(invocation, asyncResp);
@@ -367,7 +370,6 @@ public class TestFaultInjectHandler {
     Mockito.when(invocation.getSchemaId()).thenReturn("testSchema1");
     Mockito.when(invocation.getMicroserviceName()).thenReturn("carts1");
     boolean validAssert;
-    long timeOld = System.currentTimeMillis();
 
     List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
     handler.setFaultFeature(faultInjectionFeatureList);
@@ -385,15 +387,9 @@ public class TestFaultInjectHandler {
         .updateProperty("cse.governance.Consumer._global.policy.fault.protocols.rest.abort.httpStatus", 421);
 
     handler.handle(invocation, ar -> {
-      //check whether error code return, defaut is 421.
-      assertEquals(421, response.getStatusCode());
       assertEquals(true, response.isFailed());
-      long timeNow = System.currentTimeMillis();
-      //if really time delay is added it should be greater than 5s.
-      Assert.assertTrue((timeNow - timeOld) >= 500);
     });
 
-
     System.getProperties()
         .remove(
             "cse.governance.Consumer._global.policy.fault.protocols.rest.delay.fixedDelay");
@@ -617,7 +613,6 @@ public class TestFaultInjectHandler {
 
     AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName10");
     assertEquals(3, count.get());
-
   }
 
   /**
@@ -641,7 +636,6 @@ public class TestFaultInjectHandler {
     Mockito.when(invocation.getSchemaId()).thenReturn("testSchema4");
     Mockito.when(invocation.getMicroserviceName()).thenReturn("carts5");
     boolean validAssert;
-    long timeOld = System.currentTimeMillis();
 
     List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
     handler.setFaultFeature(faultInjectionFeatureList);
@@ -657,15 +651,9 @@ public class TestFaultInjectHandler {
         .updateProperty("cse.governance.Consumer.carts5.policy.fault.protocols.rest.abort.percent", 500);
 
     handler.handle(invocation, ar -> {
-      //check whether error code return,
-      assertEquals(421, response.getStatusCode());
       assertEquals(true, response.isFailed());
-      long timeNow = System.currentTimeMillis();
-      //if really time delay is added it should be greater than 5s.
-      Assert.assertTrue((timeNow - timeOld) >= 500);
     });
 
-
     System.getProperties()
         .remove("cse.governance.Consumer.carts5.policy.fault.protocols.rest.delay.fixedDelay");
     System.getProperties()
@@ -675,9 +663,43 @@ public class TestFaultInjectHandler {
     System.getProperties()
         .remove("cse.governance.Consumer.carts5.policy.fault.protocols.rest.abort.httpStatus");
 
-
     AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName11");
     assertEquals(3, count.get());
+  }
+
+  /**
+   * Tests the fault injection handler functionality with configuration change event for service level config.
+   * 
+   * @throws Exception
+   */
+  @Test
+  public void testFaultInjectHandlerConfigChangeEvent6() throws Exception {
+    System.setProperty("cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.fixedDelay", "1000");
+
+    System.setProperty(
+        "cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.percent",
+        "100");
+
+    Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName12");
+    Mockito.when(invocation.getTransport()).thenReturn(transport);
+    Mockito.when(transport.getName()).thenReturn("rest");
+    Mockito.when(invocation.getOperationName()).thenReturn("sayBye4");
+    Mockito.when(invocation.getSchemaId()).thenReturn("testSchema4");
+    Mockito.when(invocation.getMicroserviceName()).thenReturn("carts6");
+
+    DelayFault delayFault = new DelayFault();
+    FaultParam faultParam = new FaultParam(3);
+    Vertx vertx = VertxUtils.getOrCreateVertxByName("faultinjectionTest", null);
+    faultParam.setVertx(vertx);
+
+    delayFault.injectFault(invocation, faultParam, ar);
+    System.getProperties()
+        .remove("cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.fixedDelay");
+    System.getProperties()
+        .remove("cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.percent");
+
+    AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName12");
+    assertEquals(1, count.get());
 
   }
 }

-- 
To stop receiving notification emails like this one, please contact
liubao@apache.org.