You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@servicecomb.apache.org by GitBox <gi...@apache.org> on 2018/03/22 09:16:34 UTC
[GitHub] liubao68 closed pull request #615: [SCB-324] Fault-Injection handler implementation
liubao68 closed pull request #615: [SCB-324] Fault-Injection handler implementation
URL: https://github.com/apache/incubator-servicecomb-java-chassis/pull/615
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/coverage-reports/pom.xml b/coverage-reports/pom.xml
index 27a6c0e35..337ae2699 100644
--- a/coverage-reports/pom.xml
+++ b/coverage-reports/pom.xml
@@ -70,6 +70,10 @@
<groupId>org.apache.servicecomb</groupId>
<artifactId>handler-flowcontrol-qps</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.servicecomb</groupId>
+ <artifactId>handler-fault-injection</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.servicecomb</groupId>
<artifactId>common-rest</artifactId>
diff --git a/handlers/handler-fault-injection/pom.xml b/handlers/handler-fault-injection/pom.xml
new file mode 100755
index 000000000..de176d40a
--- /dev/null
+++ b/handlers/handler-fault-injection/pom.xml
@@ -0,0 +1,49 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.servicecomb</groupId>
+ <artifactId>handlers</artifactId>
+ <version>1.0.0-m2-SNAPSHOT</version>
+ </parent>
+ <artifactId>handler-fault-injection</artifactId>
+ <name>Java Chassis::Handlers::Fault Injection</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.servicecomb</groupId>
+ <artifactId>java-chassis-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.servicecomb</groupId>
+ <artifactId>foundation-test-scaffolding</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbortFault.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbortFault.java
new file mode 100644
index 000000000..1135aa6e1
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbortFault.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+import org.apache.servicecomb.swagger.invocation.exception.CommonExceptionData;
+import org.apache.servicecomb.swagger.invocation.exception.InvocationException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AbortFault extends AbstractFault {
+ private static final Logger LOGGER = LoggerFactory.getLogger(AbortFault.class);
+
+ @Override
+ public void injectFault(Invocation invocation, FaultParam faultParam, AsyncResponse asynResponse) {
+ // get the config values related to abort.
+ int abortPercent = FaultInjectionUtil.getFaultInjectionConfig(invocation,
+ "abort.percent");
+
+ if (abortPercent == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: Abort percentage is not configured");
+ asynResponse.success("success");
+ return;
+ }
+
+ // check fault abort condition.
+ boolean isAbort = FaultInjectionUtil.isFaultNeedToInject(faultParam.getReqCount(), abortPercent);
+ if (isAbort) {
+ // get the config values related to abort percentage.
+ int errorCode = FaultInjectionUtil.getFaultInjectionConfig(invocation,
+ "abort.httpStatus");
+
+ if (errorCode == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: Abort error code is not configured");
+ asynResponse.success("success");
+ return;
+ }
+ // if request need to be abort then return failure with given error code
+ CommonExceptionData errorData = new CommonExceptionData("aborted by fault inject");
+ asynResponse.consumerFail(new InvocationException(errorCode, "aborted by fault inject", errorData));
+ return;
+ }
+
+ asynResponse.success("success");
+ }
+
+ @Override
+ public int getOrder() {
+ return 200;
+ }
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbstractFault.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbstractFault.java
new file mode 100644
index 000000000..e92a68ef5
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/AbstractFault.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+public abstract class AbstractFault implements Fault {
+
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/DelayFault.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/DelayFault.java
new file mode 100644
index 000000000..ecbb2e103
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/DelayFault.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.vertx.core.Handler;
+import io.vertx.core.Vertx;
+
+public class DelayFault extends AbstractFault {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DelayFault.class);
+
+ @Override
+ public int getOrder() {
+ return 100;
+ }
+
+ @Override
+ public void injectFault(Invocation invocation, FaultParam faultParam, AsyncResponse asynResponse) {
+ int delayPercent = FaultInjectionUtil.getFaultInjectionConfig(invocation,
+ "delay.percent");
+
+ if (delayPercent == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: delay percentage is not configured");
+ asynResponse.success("success");
+ return;
+ }
+
+ // check fault delay condition.
+ boolean isDelay = FaultInjectionUtil.isFaultNeedToInject(faultParam.getReqCount(), delayPercent);
+ if (isDelay) {
+ LOGGER.debug("Fault injection: delay is added for the request by fault inject handler");
+ long delay = FaultInjectionUtil.getFaultInjectionConfig(invocation,
+ "delay.fixedDelay");
+
+ if (delay == FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE) {
+ LOGGER.debug("Fault injection: delay is not configured");
+ asynResponse.success("success");
+ return;
+ }
+
+ Vertx vertx = faultParam.getVertx();
+ if (vertx != null) {
+ vertx.setTimer(delay, new Handler<Long>() {
+ @Override
+ public void handle(Long timeID) {
+ asynResponse.success("success");
+ }
+ });
+ } else {
+ try {
+ Thread.sleep(delay);
+ } catch (InterruptedException e) {
+ LOGGER.info("Interrupted exception is received");
+ }
+ asynResponse.success("success");
+ }
+ }
+ }
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/Fault.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/Fault.java
new file mode 100644
index 000000000..ddc422953
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/Fault.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+
+public interface Fault {
+
+ int getOrder();
+
+ void injectFault(Invocation invocation, FaultParam faultAttributes, AsyncResponse asynResponse);
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultExecutor.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultExecutor.java
new file mode 100644
index 000000000..9a20f64c9
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultExecutor.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import java.util.List;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+import org.apache.servicecomb.swagger.invocation.Response;
+
+
+/**
+ * Implements the fault feature execution one after other.
+ */
+public class FaultExecutor {
+ private List<Fault> faultInjectList;
+
+ private int handlerIndex = 0;
+
+ private Invocation invocation;
+
+ private FaultParam param;
+
+ public FaultExecutor(List<Fault> faultInjectList, Invocation invocation, FaultParam param) {
+ this.faultInjectList = faultInjectList;
+ this.invocation = invocation;
+ this.param = param;
+ }
+
+ public void execute(AsyncResponse asyncResponse) {
+ this.next(asyncResponse);
+ }
+
+ private void next(AsyncResponse asyncResponse) {
+ if (handlerIndex >= faultInjectList.size()) {
+ asyncResponse.complete(Response.succResp("success"));
+ return;
+ }
+
+ int runIndex = handlerIndex;
+ handlerIndex++;
+ faultInjectList.get(runIndex).injectFault(invocation, param, response -> {
+ if (response.isFailed()) {
+ asyncResponse.complete(response);
+ } else {
+ FaultExecutor.this.next(asyncResponse);
+ }
+ });
+ }
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConfig.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConfig.java
new file mode 100755
index 000000000..d43eb2f38
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConfig.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.servicecomb.foundation.common.concurrent.ConcurrentHashMapEx;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.netflix.config.DynamicIntProperty;
+import com.netflix.config.DynamicPropertyFactory;
+
+/**
+ * Handles the fault injection configuration read from micro service file/config
+ * center.
+ */
+public final class FaultInjectionConfig {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(FaultInjectionConfig.class);
+
+ // key is configuration parameter.
+ private static Map<String, String> cfgCallback = new ConcurrentHashMapEx<>();
+
+ public static int getConfigVal(String config, int defaultValue) {
+ DynamicIntProperty dynamicIntProperty = DynamicPropertyFactory.getInstance().getIntProperty(config,
+ defaultValue);
+
+ cfgCallback.computeIfAbsent(config, key -> {
+ dynamicIntProperty.addCallback(() -> {
+ int newValue = dynamicIntProperty.get();
+ String cfgName = dynamicIntProperty.getName();
+
+ //store the value in config center map and check for next requests.
+ FaultInjectionUtil.setConfigCenterValue(cfgName, new AtomicInteger(newValue));
+ LOGGER.info("{} changed to {}", cfgName, newValue);
+ });
+ return config;
+ });
+
+ return dynamicIntProperty.get();
+ }
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConst.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConst.java
new file mode 100644
index 000000000..b1221e734
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionConst.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+/**
+ * Handles the all constant values for fault injection.
+ */
+public class FaultInjectionConst {
+
+ public static final int FAULT_INJECTION_DEFAULT_VALUE = -1;
+
+ public static final int FAULT_INJECTION_ERROR = -1;
+
+ public static final String CONSUMER_FAULTINJECTION = "cse.governance.Consumer.";
+
+ public static final String CONSUMER_FAULTINJECTION_GLOBAL = "cse.governance.Consumer._global.";
+
+ public static final String CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS = "policy.fault.protocols.";
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionHandler.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionHandler.java
new file mode 100755
index 000000000..8c2924613
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionHandler.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.servicecomb.core.Handler;
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.foundation.common.utils.SPIServiceUtils;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+
+import io.vertx.core.Context;
+import io.vertx.core.Vertx;
+
+/**
+ * Fault injection handler which injects the delay/abort for requests based on
+ * the percentage configured in service file.
+ *
+ */
+
+public class FaultInjectionHandler implements Handler {
+
+ private List<Fault> faultInjectionFeatureList = SPIServiceUtils.getSortedService(Fault.class);
+
+ //added only for unit testing
+ public void setFaultFeature(List<Fault> faultFeature) {
+ faultInjectionFeatureList = faultFeature;
+ }
+
+ @Override
+ public void handle(Invocation invocation, AsyncResponse asyncResp) throws Exception {
+
+ // prepare the key and lookup for request count.
+ String key = invocation.getTransport().getName() + invocation.getMicroserviceQualifiedName();
+ AtomicLong reqCount = FaultInjectionUtil.getOperMetTotalReq(key);
+ // increment the request count here after checking the delay/abort condition.
+ long reqCountCurrent = reqCount.getAndIncrement();
+
+ FaultParam param = new FaultParam(reqCountCurrent);
+ Context currentContext = Vertx.currentContext();
+ if (currentContext != null && currentContext.isEventLoopContext()) {
+ param.setVertx(currentContext.owner());
+ }
+
+ FaultExecutor executor = new FaultExecutor(faultInjectionFeatureList, invocation, param);
+ executor.execute(response -> {
+ try {
+ if (response.isFailed()) {
+ asyncResp.complete(response);
+ } else {
+ invocation.next(asyncResp);
+ }
+ } catch (Exception e) {
+ asyncResp.consumerFail(e);
+ }
+ });
+ }
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionUtil.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionUtil.java
new file mode 100755
index 000000000..05e85aefb
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultInjectionUtil.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import static org.apache.servicecomb.faultinjection.FaultInjectionConst.CONSUMER_FAULTINJECTION;
+import static org.apache.servicecomb.faultinjection.FaultInjectionConst.CONSUMER_FAULTINJECTION_GLOBAL;
+import static org.apache.servicecomb.faultinjection.FaultInjectionConst.CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS;
+import static org.apache.servicecomb.faultinjection.FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.foundation.common.concurrent.ConcurrentHashMapEx;
+
+/**
+ * Handles the count for all request based key[transport + microservice qualified name].
+ */
+public class FaultInjectionUtil {
+
+ // key is transport+operQualifiedName
+ private static Map<String, AtomicLong> requestCount = new ConcurrentHashMapEx<>();
+
+ // key is config paramter
+ private static Map<String, AtomicInteger> configCenterValue = new ConcurrentHashMapEx<>();
+
+ /**
+ * Returns total requests per provider for operational level.
+ *
+ * @param key
+ * transport+operational name
+ * @return long total requests
+ */
+ public static AtomicLong getOperMetTotalReq(String key) {
+ return requestCount.computeIfAbsent(key, p -> new AtomicLong(1));
+ }
+
+ /**
+ * Returns the map of config parameter key and values.
+ * @return value of config parameter map
+ */
+ public static Map<String, AtomicInteger> getConfigCenterMap() {
+ return configCenterValue;
+ }
+
+ /**
+ * Sets the value for given config parameter.
+ * @param key
+ * @param value
+ */
+ public static void setConfigCenterValue(String key, AtomicInteger value) {
+ configCenterValue.put(key, value);
+ }
+
+ /**
+ * Handles the reading fault injection configuration.
+ *
+ * @param invocation
+ * invocation of request
+ * @param key
+ * configuration key
+ * @return configuration value
+ */
+ public static int getFaultInjectionConfig(Invocation invocation, String key) {
+ int value = 0;
+ String config;
+
+ // get the config base on priority. operationName-->schema-->service-->global
+ String operationName = invocation.getOperationName();
+ String schema = invocation.getSchemaId();
+ String serviceName = invocation.getMicroserviceName();
+
+ config = CONSUMER_FAULTINJECTION + serviceName + ".schemas." + schema + ".operations." + operationName + "."
+ + CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS + invocation.getTransport().getName() + "." + key;
+
+ value = getConfigValue(config);
+ if ((value != FAULT_INJECTION_DEFAULT_VALUE)) {
+ return value;
+ }
+
+ config = CONSUMER_FAULTINJECTION + serviceName + ".schemas." + schema + "."
+ + CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS + invocation.getTransport().getName() + "." + key;
+
+ value = getConfigValue(config);
+ if ((value != FAULT_INJECTION_DEFAULT_VALUE)) {
+ return value;
+ }
+
+ config = CONSUMER_FAULTINJECTION + serviceName + "." + CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS
+ + invocation.getTransport().getName() + "." + key;
+ value = getConfigValue(config);
+ if ((value != FAULT_INJECTION_DEFAULT_VALUE)) {
+ return value;
+ }
+
+ config = CONSUMER_FAULTINJECTION_GLOBAL + CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS
+ + invocation.getTransport().getName() + "." + key;
+
+ value = getConfigValue(config);
+ return value;
+ }
+
+ /**
+ * Get the configuration value
+ * @param config config parameter
+ * @return int value
+ */
+ private static int getConfigValue(String config) {
+ int value = 0;
+ //first need to check in config center map which has high priority.
+ Map<String, AtomicInteger> cfgMap = FaultInjectionUtil.getConfigCenterMap();
+
+ if (cfgMap.containsKey(config)) {
+ return cfgMap.get(config).get();
+ }
+
+ value = FaultInjectionConfig.getConfigVal(config, FAULT_INJECTION_DEFAULT_VALUE);
+ return value;
+
+ }
+
+ /**
+ * It will check the delay/abort condition based on request count and percentage
+ * received.
+ *
+ * @param reqCount
+ * @param percentage
+ * @param key
+ * @return true: delay/abort is needed. false: delay/abort is not needed.
+ */
+ public static boolean isFaultNeedToInject(long reqCount, int percentage) {
+ /*
+ * Example: delay/abort percentage configured is 10% and Get the count(suppose
+ * if it is 10th request) from map and calculate resultNew(10th request) and
+ * requestOld(9th request). Like this for every request it will calculate
+ * current request count and previous count. if both not matched need to add
+ * delay/abort otherwise no need to add.
+ */
+
+ // calculate the value with current request count.
+ long resultNew = (reqCount * percentage) / 100;
+
+ // calculate the value with previous count value.
+ long resultOld = ((reqCount - 1) * percentage) / 100;
+
+ // if both are not matching then delay/abort should be added.
+ return (resultNew != resultOld);
+ }
+}
diff --git a/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultParam.java b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultParam.java
new file mode 100644
index 000000000..92f0e84eb
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/java/org/apache/servicecomb/faultinjection/FaultParam.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import io.vertx.core.Vertx;
+
+/**
+ * Fault injection parameters which decides the fault injection condition.
+ */
+public class FaultParam {
+ private long reqCount;
+
+ private Vertx vertx;
+
+ public long getReqCount() {
+ return reqCount;
+ }
+
+ public void setReqCount(long reqCount) {
+ this.reqCount = reqCount;
+ }
+
+ FaultParam(long reqCount) {
+ this.reqCount = reqCount;
+ }
+
+ public Vertx getVertx() {
+ return vertx;
+ }
+
+ public void setVertx(Vertx vertx) {
+ this.vertx = vertx;
+ }
+}
diff --git a/handlers/handler-fault-injection/src/main/resources/META-INF/services/org.apache.servicecomb.faultinjection.Fault b/handlers/handler-fault-injection/src/main/resources/META-INF/services/org.apache.servicecomb.faultinjection.Fault
new file mode 100644
index 000000000..1da13d5db
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/resources/META-INF/services/org.apache.servicecomb.faultinjection.Fault
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.servicecomb.faultinjection.DelayFault
+org.apache.servicecomb.faultinjection.AbortFault
diff --git a/handlers/handler-fault-injection/src/main/resources/config/cse.handler.xml b/handlers/handler-fault-injection/src/main/resources/config/cse.handler.xml
new file mode 100755
index 000000000..4c1553e8d
--- /dev/null
+++ b/handlers/handler-fault-injection/src/main/resources/config/cse.handler.xml
@@ -0,0 +1,21 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<config>
+ <handler id="fault-injection-consumer"
+ class="org.apache.servicecomb.faultinjection.FaultInjectionHandler" />
+</config>
diff --git a/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectConfig.java b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectConfig.java
new file mode 100644
index 000000000..a9ea092a8
--- /dev/null
+++ b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectConfig.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Tests the fault injection configuration.
+ */
+public class TestFaultInjectConfig {
+ FaultInjectionConfig faultCfg;
+
+ FaultInjectionConst faultConst;
+
+ FaultInjectionUtil faultUtil;
+
+ FaultParam faultParam;
+
+ AbortFault abortFault;
+
+ DelayFault delayFault;
+
+ @Before
+ public void setUp() throws Exception {
+ faultCfg = new FaultInjectionConfig();
+ faultConst = new FaultInjectionConst();
+ faultUtil = new FaultInjectionUtil();
+ faultParam = new FaultParam(10);
+ abortFault = new AbortFault();
+ delayFault = new DelayFault();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ faultCfg = null;
+ faultConst = null;
+ faultUtil = null;
+ faultParam = null;
+ }
+
+ @Test
+ public void testFaultInjectConfig() throws Exception {
+ int val = FaultInjectionConfig.getConfigVal("cse.servicecomb.handler.consumer.faultinject.config", 0);
+ Assert.assertEquals(0, val);
+ }
+
+ @Test
+ public void testConstants() {
+ assertEquals("cse.governance.Consumer.", FaultInjectionConst.CONSUMER_FAULTINJECTION);
+ assertEquals("policy.fault.protocols.", FaultInjectionConst.CONSUMER_FAULTINJECTION_POLICY_PROTOCOLS);
+ assertEquals(-1, FaultInjectionConst.FAULT_INJECTION_DEFAULT_VALUE);
+ assertEquals("cse.governance.Consumer._global.", FaultInjectionConst.CONSUMER_FAULTINJECTION_GLOBAL);
+ assertEquals(-1, FaultInjectionConst.FAULT_INJECTION_ERROR);
+ }
+
+ @Test
+ public void testFaultParam() {
+ faultParam.setReqCount(100);
+ faultParam.setVertx(null);
+ assertEquals(100, faultParam.getReqCount());
+ assertEquals(null, faultParam.getVertx());
+ }
+
+ @Test
+ public void testFaultPriority() {
+ assertEquals(200, abortFault.getOrder());
+ assertEquals(100, delayFault.getOrder());
+ }
+}
diff --git a/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectHandler.java b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectHandler.java
new file mode 100644
index 000000000..255de4399
--- /dev/null
+++ b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectHandler.java
@@ -0,0 +1,706 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.servicecomb.core.Invocation;
+import org.apache.servicecomb.core.Transport;
+import org.apache.servicecomb.core.definition.OperationMeta;
+import org.apache.servicecomb.foundation.vertx.VertxUtils;
+import org.apache.servicecomb.swagger.invocation.AsyncResponse;
+import org.apache.servicecomb.swagger.invocation.Response;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import io.vertx.core.Vertx;
+
+/**
+ * Tests the fault injection handler functionality.
+ */
+public class TestFaultInjectHandler {
+ FaultInjectionHandler handler;
+
+ Invocation invocation;
+
+ AsyncResponse asyncResp;
+
+ OperationMeta operationMeta;
+
+ private Transport transport;
+
+ Response response;
+
+ boolean isDelay;
+
+ boolean isAbort;
+
+ @InjectMocks
+ FaultInjectionHandler faultHandler;
+
+ @InjectMocks
+ AbortFault abortFault;
+
+ @InjectMocks
+ DelayFault delayFault;
+
+ AsyncResponse ar = new AsyncResponse() {
+ @Override
+ public void handle(Response resp) {
+ response = resp;
+ }
+ };
+
+
+ @Before
+ public void setUp() throws Exception {
+ handler = new FaultInjectionHandler();
+
+ invocation = Mockito.mock(Invocation.class);
+
+ asyncResp = Mockito.mock(AsyncResponse.class);
+
+ operationMeta = Mockito.mock(OperationMeta.class);
+
+ transport = Mockito.mock(Transport.class);
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ handler = null;
+
+ invocation = null;
+
+ asyncResp = null;
+
+ operationMeta = null;
+
+ transport = null;
+ }
+
+ /**
+ * Tests the fault injection handler functionality with default values for
+ * highway transport.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerHighwayWithDefaultCfg() throws Exception {
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName1");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("highway");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayHello");
+ Mockito.when(invocation.getSchemaId()).thenReturn("sayHelloSchema");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("hello");
+
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(abortFault, delayFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ handler.handle(invocation, asyncResp);
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("highwayMicroserviceQualifiedName1");
+ assertEquals(2, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with default values for rest
+ * transport.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerRestWithDefaultCfg() throws Exception {
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName2");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayHello");
+ Mockito.when(invocation.getSchemaId()).thenReturn("sayHelloSchema");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("hello");
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ handler.handle(invocation, asyncResp);
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName2");
+ assertEquals(2, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with global configuration
+ * with delay/abort condition.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerConfigChangeGlobal() throws Exception {
+
+ System.setProperty("cse.governance.Consumer._global.policy.fault.protocols.rest.delay.fixedDelay", "5");
+ System.setProperty("cse.governance.Consumer._global.policy.fault.protocols.rest.delay.percent", "10");
+ System.setProperty("cse.governance.Consumer._global.policy.fault.protocols.rest.abort.percent", "10");
+ System.setProperty("cse.governance.Consumer._global.policy.fault.protocols.rest.abort.httpStatus", "421");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName3");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayHello");
+ Mockito.when(invocation.getSchemaId()).thenReturn("sayHelloSchema");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("hello");
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(abortFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ handler.handle(invocation, ar -> {
+ //this case no delay/no abort so reponse is null, it should not enter this in this block.
+ isDelay = true;
+ isAbort = true;
+ });
+ Assert.assertFalse(isDelay);
+ Assert.assertFalse(isAbort);
+
+ System.getProperties().remove("cse.governance.Consumer._global.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties().remove("cse.governance.Consumer._global.policy.fault.protocols.rest.delay.percent");
+ System.getProperties().remove("cse.governance.Consumer._global.policy.fault.protocols.rest.abort.percent");
+ System.getProperties().remove("cse.governance.Consumer._global.policy.fault.protocols.rest.abort.httpStatus");
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName3");
+ assertEquals(2, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with service level configuration
+ * with delay/abort condition.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerServiceCfgSuccess() throws Exception {
+
+ System.setProperty("cse.governance.Consumer.carts.policy.fault.protocols.rest.delay.fixedDelay", "1");
+ System.setProperty("cse.governance.Consumer.carts.policy.fault.protocols.rest.delay.percent", "10");
+ System.setProperty("cse.governance.Consumer.carts.policy.fault.protocols.rest.abort.percent", "10");
+ System.setProperty("cse.governance.Consumer.carts.policy.fault.protocols.rest.abort.httpStatus", "421");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName4");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayHello");
+ Mockito.when(invocation.getSchemaId()).thenReturn("sayHelloSchema");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts");
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(abortFault, delayFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ handler.handle(invocation, ar -> {
+ //this case no delay/no abort so reponse is null, it should not enter this in this block.
+ isDelay = true;
+ isAbort = true;
+ });
+ Assert.assertFalse(isDelay);
+ Assert.assertFalse(isAbort);
+
+ System.getProperties().remove("cse.governance.Consumer.carts.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties().remove("cse.governance.Consumer.carts.policy.fault.protocols.rest.delay.percent");
+ System.getProperties().remove("cse.governance.Consumer.carts.policy.fault.protocols.rest.abort.percent");
+ System.getProperties().remove("cse.governance.Consumer.carts.policy.fault.protocols.rest.abort.httpStatus");
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName4");
+ assertEquals(2, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with schema level configuration
+ * with delay/abort condition.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerSchemaCfgSuccess() throws Exception {
+
+ System.setProperty("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.delay.fixedDelay",
+ "1");
+ System.setProperty("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.delay.percent",
+ "10");
+ System.setProperty("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.abort.percent",
+ "10");
+ System.setProperty("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.abort.httpStatus",
+ "421");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName5");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayHello");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts");
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(abortFault, delayFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ handler.handle(invocation, ar -> {
+ //this case no delay/no abort so reponse is null, it should not enter this in this block.
+ isDelay = true;
+ isAbort = true;
+ });
+ Assert.assertFalse(isDelay);
+ Assert.assertFalse(isAbort);
+
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.delay.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.abort.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts.schemas.testSchema.policy.fault.protocols.rest.abort.httpStatus");
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName5");
+ assertEquals(2, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with operation level configuration
+ * with delay/abort condition.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerOperationCfgSuccess() throws Exception {
+
+ System.setProperty(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.delay.fixedDelay",
+ "1");
+ System.setProperty(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.delay.percent",
+ "10");
+ System.setProperty(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.abort.percent",
+ "10");
+ System.setProperty(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.abort.httpStatus",
+ "421");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName6");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayHi");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts");
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(abortFault, delayFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ handler.handle(invocation, ar -> {
+ //this case no delay/no abort so reponse is null, it should not enter this in this block.
+ isDelay = true;
+ isAbort = true;
+ });
+
+ Assert.assertFalse(isDelay);
+ Assert.assertFalse(isAbort);
+
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.delay.percent");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.abort.percent");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts.schemas.testSchema.operations.sayHi.policy.fault.protocols.rest.abort.httpStatus");
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName6");
+ assertEquals(2, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with configuration change event for global level config.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerConfigChangeEvent1() throws Exception {
+
+ System.setProperty(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.delay.fixedDelay",
+ "1");
+ System.setProperty(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.delay.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.abort.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.abort.httpStatus",
+ "420");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName7");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayBye1");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema1");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts1");
+ boolean validAssert;
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ try {
+ validAssert = true;
+ handler.handle(invocation, ar);
+ } catch (Exception e) {
+ validAssert = false;
+ }
+ Assert.assertTrue(validAssert);
+ TestFaultInjectUtil
+ .updateProperty("cse.governance.Consumer._global.policy.fault.protocols.rest.delay.fixedDelay", 500);
+ TestFaultInjectUtil
+ .updateProperty("cse.governance.Consumer._global.policy.fault.protocols.rest.abort.httpStatus", 421);
+
+ handler.handle(invocation, ar -> {
+ assertEquals(true, response.isFailed());
+ });
+
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.delay.percent");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.abort.percent");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer._global.policy.fault.protocols.rest.abort.httpStatus");
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName7");
+ assertEquals(3, count.get());
+
+ }
+
+ /**
+ * Tests the fault injection handler functionality with configuration change event for operation level config.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerConfigChangeEvent2() throws Exception {
+
+ System.setProperty(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.delay.fixedDelay",
+ "1");
+ System.setProperty(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.delay.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.abort.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.abort.httpStatus",
+ "420");
+
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName8");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayBye2");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema2");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts2");
+ boolean validAssert;
+ long timeOld = System.currentTimeMillis();
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ try {
+ validAssert = true;
+ handler.handle(invocation, ar);
+ } catch (Exception e) {
+ validAssert = false;
+ }
+ Assert.assertTrue(validAssert);
+ TestFaultInjectUtil
+ .updateProperty(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.delay.fixedDelay",
+ 500);
+
+ handler.handle(invocation, ar -> {
+ //check whether error code return
+ assertEquals(420, response.getStatusCode());
+ assertEquals(true, response.isFailed());
+ long timeNow = System.currentTimeMillis();
+ //if really time delay is added it should be greater than 5s.
+ Assert.assertTrue((timeNow - timeOld) >= 500);
+ });
+
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.delay.percent");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.abort.percent");
+ System.getProperties()
+ .remove(
+ "cse.governance.Consumer.carts2.schemas.testSchema2.operations.sayBye2.policy.fault.protocols.rest.abort.httpStatus");
+
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName8");
+ assertEquals(3, count.get());
+
+ }
+
+ /**
+ * Tests the fault injection handler functionality with configuration change event for schema level config.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerConfigChangeEvent3() throws Exception {
+
+ System.setProperty(
+ "cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.delay.fixedDelay",
+ "1");
+ System.setProperty(
+ "cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.delay.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.abort.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.abort.httpStatus",
+ "421");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName9");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayBye3");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema3");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts3");
+ boolean validAssert;
+ long timeOld = System.currentTimeMillis();
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ try {
+ validAssert = true;
+ handler.handle(invocation, ar);
+ } catch (Exception e) {
+ validAssert = false;
+ }
+ Assert.assertTrue(validAssert);
+ TestFaultInjectUtil
+ .updateProperty(
+ "cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.delay.fixedDelay",
+ 500);
+
+ handler.handle(invocation, ar -> {
+ //check whether error code return, defaut is 421.
+ assertEquals(421, response.getStatusCode());
+ assertEquals(true, response.isFailed());
+ long timeNow = System.currentTimeMillis();
+ //if really time delay is added it should be greater than 5s.
+ Assert.assertTrue((timeNow - timeOld) >= 500);
+ });
+
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.delay.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.abort.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts3.schemas.testSchema3.policy.fault.protocols.rest.abort.httpStatus");
+
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName9");
+ assertEquals(3, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with configuration change event for service level config.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerConfigChangeEvent4() throws Exception {
+ System.setProperty("cse.governance.Consumer.carts4.policy.fault.protocols.rest.delay.fixedDelay", "1");
+
+ System.setProperty(
+ "cse.governance.Consumer.carts4.policy.fault.protocols.rest.delay.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer.carts4.policy.fault.protocols.rest.abort.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer.carts4.policy.fault.protocols.rest.abort.httpStatus",
+ "421");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName10");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayBye4");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema4");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts4");
+ boolean validAssert;
+ long timeOld = System.currentTimeMillis();
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ try {
+ validAssert = true;
+ handler.handle(invocation, ar);
+ } catch (Exception e) {
+ validAssert = false;
+ }
+ Assert.assertTrue(validAssert);
+ TestFaultInjectUtil
+ .updateProperty("cse.governance.Consumer.carts4.policy.fault.protocols.rest.delay.fixedDelay", 500);
+
+ handler.handle(invocation, ar -> {
+ //check whether error code return,
+ assertEquals(421, response.getStatusCode());
+ assertEquals(true, response.isFailed());
+ long timeNow = System.currentTimeMillis();
+ //if really time delay is added it should be greater than 5s.
+ Assert.assertTrue((timeNow - timeOld) >= 500);
+ });
+
+
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts4.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts4.policy.fault.protocols.rest.delay.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts4.policy.fault.protocols.rest.abort.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts4.policy.fault.protocols.rest.abort.httpStatus");
+
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName10");
+ assertEquals(3, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with configuration change event for service level config.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerConfigChangeEvent5() throws Exception {
+ System.setProperty(
+ "cse.governance.Consumer.carts5.policy.fault.protocols.rest.delay.percent",
+ "100");
+ System.setProperty(
+ "cse.governance.Consumer.carts5.policy.fault.protocols.rest.abort.percent",
+ "100");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName11");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayBye4");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema4");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts5");
+ boolean validAssert;
+
+ List<Fault> faultInjectionFeatureList = Arrays.asList(delayFault, abortFault);
+ handler.setFaultFeature(faultInjectionFeatureList);
+
+ try {
+ validAssert = true;
+ handler.handle(invocation, ar);
+ } catch (Exception e) {
+ validAssert = false;
+ }
+ Assert.assertTrue(validAssert);
+ TestFaultInjectUtil
+ .updateProperty("cse.governance.Consumer.carts5.policy.fault.protocols.rest.abort.percent", 500);
+
+ handler.handle(invocation, ar -> {
+ assertEquals(true, response.isFailed());
+ });
+
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts5.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts5.policy.fault.protocols.rest.delay.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts5.policy.fault.protocols.rest.abort.percent");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts5.policy.fault.protocols.rest.abort.httpStatus");
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName11");
+ assertEquals(3, count.get());
+ }
+
+ /**
+ * Tests the fault injection handler functionality with configuration change event for service level config.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFaultInjectHandlerConfigChangeEvent6() throws Exception {
+ System.setProperty("cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.fixedDelay", "1000");
+
+ System.setProperty(
+ "cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.percent",
+ "100");
+
+ Mockito.when(invocation.getMicroserviceQualifiedName()).thenReturn("MicroserviceQualifiedName12");
+ Mockito.when(invocation.getTransport()).thenReturn(transport);
+ Mockito.when(transport.getName()).thenReturn("rest");
+ Mockito.when(invocation.getOperationName()).thenReturn("sayBye4");
+ Mockito.when(invocation.getSchemaId()).thenReturn("testSchema4");
+ Mockito.when(invocation.getMicroserviceName()).thenReturn("carts6");
+
+ DelayFault delayFault = new DelayFault();
+ FaultParam faultParam = new FaultParam(3);
+ Vertx vertx = VertxUtils.getOrCreateVertxByName("faultinjectionTest", null);
+ faultParam.setVertx(vertx);
+
+ delayFault.injectFault(invocation, faultParam, ar);
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.fixedDelay");
+ System.getProperties()
+ .remove("cse.governance.Consumer.carts6.policy.fault.protocols.rest.delay.percent");
+
+ AtomicLong count = FaultInjectionUtil.getOperMetTotalReq("restMicroserviceQualifiedName12");
+ assertEquals(1, count.get());
+
+ }
+}
diff --git a/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectUtil.java b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectUtil.java
new file mode 100644
index 000000000..ec619a7d9
--- /dev/null
+++ b/handlers/handler-fault-injection/src/test/java/org/apache/servicecomb/faultinjection/TestFaultInjectUtil.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.servicecomb.faultinjection;
+
+import java.lang.reflect.Method;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.springframework.util.ReflectionUtils;
+
+import com.netflix.config.DynamicProperty;
+
+/**
+ * Tests the fault inject util functionality.
+ */
+public class TestFaultInjectUtil {
+ private static Method updatePropertyMethod =
+ ReflectionUtils.findMethod(DynamicProperty.class, "updateProperty", String.class, Object.class);
+
+ static {
+ updatePropertyMethod.setAccessible(true);
+ }
+
+ DelayFault delayFault = Mockito.mock(DelayFault.class);
+
+ AbortFault abortFault = Mockito.mock(AbortFault.class);
+
+ public static void updateProperty(String key, Object value) {
+ ReflectionUtils.invokeMethod(updatePropertyMethod, null, key, value);
+ }
+
+ @Test
+ public void testFaultInjectUtil() throws Exception {
+ AtomicLong count1 = FaultInjectionUtil.getOperMetTotalReq("test");
+ Assert.assertEquals(1, count1.get());
+ count1.incrementAndGet();
+ AtomicLong count2 = FaultInjectionUtil.getOperMetTotalReq("test");
+ Assert.assertEquals(2, count2.get());
+ FaultInjectionUtil.setConfigCenterValue("sayHi", new AtomicInteger(123));
+ int value = FaultInjectionUtil.getConfigCenterMap().get("sayHi").get();
+ Assert.assertEquals(123, value);
+ }
+}
diff --git a/handlers/pom.xml b/handlers/pom.xml
index 2afaa7759..a16e93e44 100644
--- a/handlers/pom.xml
+++ b/handlers/pom.xml
@@ -36,6 +36,7 @@
<module>handler-bizkeeper</module>
<module>handler-flowcontrol-qps</module>
<module>handler-loadbalance</module>
+ <module>handler-fault-injection</module>
<module>handler-publickey-auth</module>
</modules>
diff --git a/java-chassis-dependencies/pom.xml b/java-chassis-dependencies/pom.xml
index 57932965e..e9ae9ecfc 100644
--- a/java-chassis-dependencies/pom.xml
+++ b/java-chassis-dependencies/pom.xml
@@ -834,6 +834,11 @@
<artifactId>handler-publickey-auth</artifactId>
<version>1.0.0-m2-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.servicecomb</groupId>
+ <artifactId>handler-fault-injection</artifactId>
+ <version>1.0.0-m2-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.servicecomb</groupId>
<artifactId>common-rest</artifactId>
diff --git a/java-chassis-distribution/pom.xml b/java-chassis-distribution/pom.xml
index 1b7f364d8..95b30e1a0 100644
--- a/java-chassis-distribution/pom.xml
+++ b/java-chassis-distribution/pom.xml
@@ -128,6 +128,10 @@
<groupId>org.apache.servicecomb</groupId>
<artifactId>handler-tracing-zipkin</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.servicecomb</groupId>
+ <artifactId>handler-fault-injection</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.servicecomb</groupId>
<artifactId>provider-jaxrs</artifactId>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services