You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cxf.apache.org by re...@apache.org on 2015/12/08 01:25:33 UTC

cxf git commit: CXF-6622: Enhance Failover Feature to support Circuit Breakers based implementation. Added full-fledged JAX-WS support.

Repository: cxf
Updated Branches:
  refs/heads/master ac61bc177 -> 111d52fd4


CXF-6622: Enhance Failover Feature to support Circuit Breakers based implementation. Added full-fledged JAX-WS support.


Project: http://git-wip-us.apache.org/repos/asf/cxf/repo
Commit: http://git-wip-us.apache.org/repos/asf/cxf/commit/111d52fd
Tree: http://git-wip-us.apache.org/repos/asf/cxf/tree/111d52fd
Diff: http://git-wip-us.apache.org/repos/asf/cxf/diff/111d52fd

Branch: refs/heads/master
Commit: 111d52fd4b92d0222af9bf1c22156145ea01ed01
Parents: ac61bc1
Author: reta <dr...@gmail.com>
Authored: Mon Dec 7 19:25:17 2015 -0500
Committer: reta <dr...@gmail.com>
Committed: Mon Dec 7 19:25:17 2015 -0500

----------------------------------------------------------------------
 .../CircuitBreakerTargetSelector.java           | 115 ++++++++++++++-----
 .../cxf/clustering/FailoverTargetSelector.java  |  43 ++++---
 .../CircuitBreakerFailoverFeature.java          |   8 ++
 ...cuitBreakerFailoverBeanDefinitionParser.java |  38 ++++++
 .../cxf/clustering/spring/NamespaceHandler.java |   2 +
 systests/uncategorized/pom.xml                  |   5 +
 .../cxf/systest/clustering/ControlImpl.java     |   1 +
 .../cxf/systest/clustering/FailoverTest.java    |  26 ++++-
 .../cxf/systest/clustering/GreeterImplE.java    |  43 +++++++
 .../apache/cxf/systest/clustering/failover.xml  |   9 ++
 .../main/resources/wsdl/greeter_control.wsdl    |   3 +
 11 files changed, 245 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/rt/features/clustering/src/main/java/org/apache/cxf/clustering/CircuitBreakerTargetSelector.java
----------------------------------------------------------------------
diff --git a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/CircuitBreakerTargetSelector.java b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/CircuitBreakerTargetSelector.java
index 62541d9..bf84e20 100644
--- a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/CircuitBreakerTargetSelector.java
+++ b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/CircuitBreakerTargetSelector.java
@@ -19,7 +19,7 @@
 
 package org.apache.cxf.clustering;
 
-import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -44,6 +44,26 @@ public class CircuitBreakerTargetSelector extends FailoverTargetSelector {
 
     private static final String IS_SELECTED = "org.apache.cxf.clustering.CircuitBreakerTargetSelector.IS_SELECTED";
     private static final Logger LOG = LogUtils.getL7dLogger(CircuitBreakerTargetSelector.class);
+
+    /**
+     * Static instance of empty (or noop) circuit breaker to handle use cases
+     * when alternative addresses or alternative endpoint addresses are nullable 
+     * (or non-valid).
+     */
+    private static final  CircuitBreaker NOOP_CIRCUIT_BREAKER = new CircuitBreaker() {
+        @Override
+        public boolean allowRequest() {
+            return true;
+        }
+        
+        @Override
+        public void markFailure(Throwable cause) {
+        }
+        
+        @Override
+        public void markSuccess() {
+        }
+    };
     
     private final int threshold;
     private final long timeout; 
@@ -65,19 +85,19 @@ public class CircuitBreakerTargetSelector extends FailoverTargetSelector {
         
         // Registering the original endpoint in the list of circuit breakers
         if (getEndpoint() != null) {
-            circuits.putIfAbsent(
-                getEndpoint().getEndpointInfo().getAddress(), 
-                new ZestCircuitBreaker(threshold, timeout)
-            );
+            final String address = getEndpoint().getEndpointInfo().getAddress();
+            if (!StringUtils.isEmpty(address)) {
+                circuits.putIfAbsent(address, new ZestCircuitBreaker(threshold, timeout));
+            }
         }
         
         if (strategy != null) {
-            for (String alternative: strategy.getAlternateAddresses(null /* no Exchange at this point */)) {
-                if (!StringUtils.isEmpty(alternative)) {
-                    circuits.putIfAbsent(
-                        alternative, 
-                        new ZestCircuitBreaker(threshold, timeout)
-                    );
+            final List<String> alternatives = strategy.getAlternateAddresses(null /* no Exchange at this point */);
+            if (alternatives != null) {
+                for (String alternative: alternatives) {
+                    if (!StringUtils.isEmpty(alternative)) {
+                        circuits.putIfAbsent(alternative, new ZestCircuitBreaker(threshold, timeout));
+                    }
                 }
             }
         }
@@ -121,15 +141,22 @@ public class CircuitBreakerTargetSelector extends FailoverTargetSelector {
             return null;
         }
         
-        final List<String> alternateAddresses = new ArrayList<>();
-        for (final Map.Entry<String, CircuitBreaker> entry: circuits.entrySet()) {
-            if (entry.getValue().allowRequest()) {
-                alternateAddresses.add(entry.getKey());
+        final List<String> alternateAddresses = updateContextAlternatives(exchange, invocation);
+        if (alternateAddresses != null) {
+            final Iterator<String> alternateAddressIterator = alternateAddresses.iterator();
+            
+            while (alternateAddressIterator.hasNext()) {
+                final String alternateAddress = alternateAddressIterator.next();
+                final CircuitBreaker circuitBreaker = getCircuitBreaker(alternateAddress);
+                
+                if (!circuitBreaker.allowRequest()) {
+                    alternateAddressIterator.remove();
+                }
             }
         }
 
         Endpoint failoverTarget = null;
-        if (!alternateAddresses.isEmpty()) {
+        if (alternateAddresses != null && !alternateAddresses.isEmpty()) {
             final String alternateAddress = getStrategy().selectAlternateAddress(alternateAddresses);
             
             // Reuse current endpoint
@@ -137,7 +164,23 @@ public class CircuitBreakerTargetSelector extends FailoverTargetSelector {
                 failoverTarget = getEndpoint();
                 failoverTarget.getEndpointInfo().setAddress(alternateAddress);
             }
-        } 
+        } else {
+            final List<Endpoint> alternateEndpoints = invocation.getAlternateEndpoints();
+            
+            if (alternateEndpoints != null) {
+                final Iterator<Endpoint> alternateEndpointIterator = alternateEndpoints.iterator();
+                
+                while (alternateEndpointIterator.hasNext()) {
+                    final Endpoint endpoint = alternateEndpointIterator.next();
+                    final CircuitBreaker circuitBreaker = getCircuitBreaker(endpoint);
+                    if (!circuitBreaker.allowRequest()) {
+                        alternateEndpointIterator.remove();
+                    }
+                }
+            }
+            
+            failoverTarget = getStrategy().selectAlternateEndpoint(alternateEndpoints);
+        }
         
         return failoverTarget;
     }
@@ -156,10 +199,7 @@ public class CircuitBreakerTargetSelector extends FailoverTargetSelector {
         
         if (requestContext != null) {
             final String address = (String)requestContext.get(Message.ENDPOINT_ADDRESS);
-            final CircuitBreaker circuitBreaker = circuits.get(address);
-            if (circuitBreaker != null) {
-                circuitBreaker.markFailure(ex);
-            }
+            getCircuitBreaker(address).markFailure(ex);
         }
     }
     
@@ -172,15 +212,36 @@ public class CircuitBreakerTargetSelector extends FailoverTargetSelector {
         
         if (requestContext != null) {
             final String address = (String)requestContext.get(Message.ENDPOINT_ADDRESS);
-            final CircuitBreaker circuitBreaker = circuits.get(address);
-            if (circuitBreaker != null) {
-                circuitBreaker.markSuccess();
+            getCircuitBreaker(address).markSuccess();
+        }
+    }
+    
+    private CircuitBreaker getCircuitBreaker(final Endpoint endpoint) {
+        return getCircuitBreaker(endpoint.getEndpointInfo().getAddress());
+    }
+    
+    private synchronized CircuitBreaker getCircuitBreaker(final String alternateAddress) {
+        CircuitBreaker circuitBreaker =  NOOP_CIRCUIT_BREAKER;
+        
+        if (!StringUtils.isEmpty(alternateAddress)) {
+            for (Map.Entry<String, CircuitBreaker> entry: circuits.entrySet()) {
+                if (alternateAddress.startsWith(entry.getKey())) {
+                    circuitBreaker = entry.getValue();
+                    break;
+                }
+            }
+            
+            if (circuitBreaker == null) {
+                circuitBreaker = new ZestCircuitBreaker(threshold, timeout);
+                circuits.put(alternateAddress, circuitBreaker);
             }
         }
+        
+        return circuitBreaker;
     }
     
     private boolean isEndpointChanged(final String address, final Endpoint target) {
-        if (address != null) {
+        if (!StringUtils.isEmpty(address)) {
             return !address.startsWith(target.getEndpointInfo().getAddress());
         } 
         
@@ -192,8 +253,8 @@ public class CircuitBreakerTargetSelector extends FailoverTargetSelector {
             target.getEndpointInfo().getAddress());
     }
     
-    protected boolean isFailoverRequired(final String address) {
-        if (address != null) {
+    private boolean isFailoverRequired(final String address) {
+        if (!StringUtils.isEmpty(address)) {
             for (final Map.Entry<String, CircuitBreaker> entry: circuits.entrySet()) {
                 if (address.startsWith(entry.getKey())) {
                     return !entry.getValue().allowRequest();

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/rt/features/clustering/src/main/java/org/apache/cxf/clustering/FailoverTargetSelector.java
----------------------------------------------------------------------
diff --git a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/FailoverTargetSelector.java b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/FailoverTargetSelector.java
index 21f129e..ce39c08 100644
--- a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/FailoverTargetSelector.java
+++ b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/FailoverTargetSelector.java
@@ -301,22 +301,7 @@ public class FailoverTargetSelector extends AbstractConduitSelector {
      */
     protected Endpoint getFailoverTarget(Exchange exchange,
                                        InvocationContext invocation) {
-        List<String> alternateAddresses = null;
-        if (!invocation.hasAlternates()) {
-            // no previous failover attempt on this invocation
-            //
-            alternateAddresses = 
-                getStrategy().getAlternateAddresses(exchange);
-            if (alternateAddresses != null) {
-                invocation.setAlternateAddresses(alternateAddresses);
-            } else {
-                invocation.setAlternateEndpoints(
-                    getStrategy().getAlternateEndpoints(exchange));
-            }
-        } else {
-            alternateAddresses = invocation.getAlternateAddresses();
-        }
-
+        List<String> alternateAddresses = updateContextAlternatives(exchange, invocation);
         Endpoint failoverTarget = null;
         if (alternateAddresses != null) {
             String alternateAddress = 
@@ -334,6 +319,32 @@ public class FailoverTargetSelector extends AbstractConduitSelector {
         }
         return failoverTarget;
     }
+
+    /**
+     * Fetches and updates the alternative address or/and alternative endpoints 
+     * (depending on the strategy) for current invocation context.
+     * @param exchange the current Exchange
+     * @param invocation the current InvocationContext
+     * @return alternative addresses
+     */
+    protected List<String> updateContextAlternatives(Exchange exchange, InvocationContext invocation) {
+        List<String> alternateAddresses = null;
+        if (!invocation.hasAlternates()) {
+            // no previous failover attempt on this invocation
+            //
+            alternateAddresses = 
+                getStrategy().getAlternateAddresses(exchange);
+            if (alternateAddresses != null) {
+                invocation.setAlternateAddresses(alternateAddresses);
+            } else {
+                invocation.setAlternateEndpoints(
+                    getStrategy().getAlternateEndpoints(exchange));
+            }
+        } else {
+            alternateAddresses = invocation.getAlternateAddresses();
+        }
+        return alternateAddresses;
+    }
     
     /**
      * Override the ENDPOINT_ADDRESS property in the request context

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/rt/features/clustering/src/main/java/org/apache/cxf/clustering/circuitbreaker/CircuitBreakerFailoverFeature.java
----------------------------------------------------------------------
diff --git a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/circuitbreaker/CircuitBreakerFailoverFeature.java b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/circuitbreaker/CircuitBreakerFailoverFeature.java
index 5ba5eb3..ef20f3c 100644
--- a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/circuitbreaker/CircuitBreakerFailoverFeature.java
+++ b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/circuitbreaker/CircuitBreakerFailoverFeature.java
@@ -55,4 +55,12 @@ public class CircuitBreakerFailoverFeature extends FailoverFeature {
     public long getTimeout() {
         return timeout;
     }
+    
+    public void setThreshold(int threshold) {
+        this.threshold = threshold;
+    }
+    
+    public void setTimeout(long timeout) {
+        this.timeout = timeout;
+    }
 }

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/CircuitBreakerFailoverBeanDefinitionParser.java
----------------------------------------------------------------------
diff --git a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/CircuitBreakerFailoverBeanDefinitionParser.java b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/CircuitBreakerFailoverBeanDefinitionParser.java
new file mode 100644
index 0000000..d3a8288
--- /dev/null
+++ b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/CircuitBreakerFailoverBeanDefinitionParser.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cxf.clustering.spring;
+
+import org.w3c.dom.Element;
+
+import org.apache.cxf.clustering.circuitbreaker.CircuitBreakerFailoverFeature;
+import org.apache.cxf.configuration.spring.AbstractBeanDefinitionParser;
+import org.springframework.beans.factory.support.BeanDefinitionBuilder;
+import org.springframework.beans.factory.xml.ParserContext;
+
+public class CircuitBreakerFailoverBeanDefinitionParser extends AbstractBeanDefinitionParser {
+    @Override
+    protected Class<?> getBeanClass(Element element) {
+        return CircuitBreakerFailoverFeature.class;
+    }
+ 
+    @Override
+    protected void mapElement(ParserContext ctx, BeanDefinitionBuilder bean, Element e, String name) {
+        setFirstChildAsProperty(e, ctx, bean, name);
+    }
+}

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/NamespaceHandler.java
----------------------------------------------------------------------
diff --git a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/NamespaceHandler.java b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/NamespaceHandler.java
index fcaed5d..156356b 100644
--- a/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/NamespaceHandler.java
+++ b/rt/features/clustering/src/main/java/org/apache/cxf/clustering/spring/NamespaceHandler.java
@@ -26,5 +26,7 @@ public class NamespaceHandler extends NamespaceHandlerSupport {
                                      new FailoverBeanDefinitionParser());
         registerBeanDefinitionParser("loadDistributor",
                                      new LoadDistributorBeanDefinitionParser());
+        registerBeanDefinitionParser("circuit-breaker-failover",
+                                     new CircuitBreakerFailoverBeanDefinitionParser());        
     }
 }

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/systests/uncategorized/pom.xml
----------------------------------------------------------------------
diff --git a/systests/uncategorized/pom.xml b/systests/uncategorized/pom.xml
index b8d1b0e..c9fd693 100644
--- a/systests/uncategorized/pom.xml
+++ b/systests/uncategorized/pom.xml
@@ -374,6 +374,11 @@
             </exclusions>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.qi4j.library</groupId>
+            <artifactId>org.qi4j.library.circuitbreaker</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <profiles>
         <profile>

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/ControlImpl.java
----------------------------------------------------------------------
diff --git a/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/ControlImpl.java b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/ControlImpl.java
index 2693f7d..e3e0351 100644
--- a/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/ControlImpl.java
+++ b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/ControlImpl.java
@@ -55,6 +55,7 @@ public class ControlImpl implements Control {
         implementors.put(FailoverTest.REPLICA_B, new GreeterImplB());
         implementors.put(FailoverTest.REPLICA_C, new GreeterImplC());
         implementors.put(FailoverTest.REPLICA_D, new GreeterImplD());
+        implementors.put(FailoverTest.REPLICA_E, new GreeterImplE());
         endpoints = new HashMap<String, Endpoint>();
     }
     

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/FailoverTest.java
----------------------------------------------------------------------
diff --git a/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/FailoverTest.java b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/FailoverTest.java
index 7412858..d548f0b 100644
--- a/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/FailoverTest.java
+++ b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/FailoverTest.java
@@ -66,7 +66,7 @@ public class FailoverTest extends AbstractBusClientServerTestBase {
     public static final String PORT_B = allocatePort(Server.class, 2);
     public static final String PORT_C = allocatePort(Server.class, 3);
     public static final String PORT_D = allocatePort(Server.class, 4);
-    public static final String PORT_EXTRA = allocatePort(Server.class, 99);
+    public static final String PORT_E = allocatePort(Server.class, 5);
     
     
     protected static final String REPLICA_A =
@@ -76,7 +76,9 @@ public class FailoverTest extends AbstractBusClientServerTestBase {
     protected static final String REPLICA_C =
         "http://localhost:" + PORT_C + "/SoapContext/ReplicatedPortC"; 
     protected static final String REPLICA_D =
-        "http://localhost:" + PORT_D + "/SoapContext/ReplicatedPortD"; 
+        "http://localhost:" + PORT_D + "/SoapContext/ReplicatedPortD";
+    protected static final String REPLICA_E =
+        "http://localhost:" + PORT_E + "/SoapContext/ReplicatedPortE";
     private static final Logger LOG =
         LogUtils.getLogger(FailoverTest.class);
     private static final String FAILOVER_CONFIG =
@@ -111,6 +113,7 @@ public class FailoverTest extends AbstractBusClientServerTestBase {
         updateWsdlExtensors("9051", PORT_A);
         updateWsdlExtensors("9052", PORT_B);
         updateWsdlExtensors("9053", PORT_C);
+        updateWsdlExtensors("9055", PORT_E);
     }
     
     @After
@@ -331,6 +334,12 @@ public class FailoverTest extends AbstractBusClientServerTestBase {
     public void testRandomStrategy() throws Exception {
         strategyTest(REPLICA_A, REPLICA_B, REPLICA_C, true);
     }
+    
+    @Test
+    public void testDefaultSequentialStrategyWithCircuitBreaker() throws Exception {
+        strategyTest(REPLICA_B, REPLICA_C, REPLICA_E, false);
+    }
+ 
     protected Greeter getGreeter(String type) throws Exception {
         if (REPLICA_A.equals(type)) {
             Greeter g = new ClusteredGreeterService().getReplicatedPortA();
@@ -342,10 +351,16 @@ public class FailoverTest extends AbstractBusClientServerTestBase {
             updateAddressPort(g, PORT_B);
             updateWsdlExtensors("9052", PORT_B);
             return g;
+        } else if (REPLICA_C.equals(type)) {
+            Greeter g = new ClusteredGreeterService().getReplicatedPortC();
+            updateAddressPort(g, PORT_C);
+            updateWsdlExtensors("9053", PORT_C);
+            return g;
         }
-        Greeter g = new ClusteredGreeterService().getReplicatedPortC();
-        updateAddressPort(g, PORT_C);
-        updateWsdlExtensors("9053", PORT_C);
+        
+        Greeter g = new ClusteredGreeterService().getReplicatedPortE();
+        updateAddressPort(g, PORT_E);
+        updateWsdlExtensors("9055", PORT_E);
         return g;
     }
     protected void strategyTest(String activeReplica1,
@@ -419,6 +434,7 @@ public class FailoverTest extends AbstractBusClientServerTestBase {
         updateWsdlExtensors("9051", PORT_A);
         updateWsdlExtensors("9052", PORT_B);
         updateWsdlExtensors("9053", PORT_C);
+        updateWsdlExtensors("9055", PORT_E);
     }
         
     protected void verifyStrategy(Object proxy, Class<?> clz) {

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/GreeterImplE.java
----------------------------------------------------------------------
diff --git a/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/GreeterImplE.java b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/GreeterImplE.java
new file mode 100644
index 0000000..74fa401
--- /dev/null
+++ b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/GreeterImplE.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cxf.systest.clustering;
+
+import javax.jws.WebService;
+
+import org.apache.cxf.greeter_control.AbstractGreeterImpl;
+
+@WebService(serviceName = "GreeterService",
+            portName = "ReplicatedPortE",
+            endpointInterface = "org.apache.cxf.greeter_control.Greeter",
+            targetNamespace = "http://cxf.apache.org/greeter_control",
+            wsdlLocation = "testutils/greeter_control.wsdl")
+public class GreeterImplE extends AbstractGreeterImpl {
+
+    private String address;
+    
+    GreeterImplE() {
+        address = FailoverTest.REPLICA_E;    
+    }
+    
+    public String greetMe(String s) {
+        return super.greetMe(s)
+               + " from: " + address;
+    }
+}

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/failover.xml
----------------------------------------------------------------------
diff --git a/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/failover.xml b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/failover.xml
index 089c533..7e761e0 100644
--- a/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/failover.xml
+++ b/systests/uncategorized/src/test/java/org/apache/cxf/systest/clustering/failover.xml
@@ -43,6 +43,15 @@
             </clustering:failover>
         </jaxws:features>
     </jaxws:client>
+    <jaxws:client name="{http://cxf.apache.org/greeter_control}ReplicatedPortE" createdFromAPI="true">
+        <jaxws:features>
+            <clustering:circuit-breaker-failover threshold="1" timeout="60000">
+                <clustering:strategy>
+                    <ref bean="Sequential"/>
+                </clustering:strategy>
+            </clustering:circuit-breaker-failover>
+        </jaxws:features>
+    </jaxws:client>
     <http:conduit name="http://localhost:.*">
         <http:client Connection="close"/>
     </http:conduit>

http://git-wip-us.apache.org/repos/asf/cxf/blob/111d52fd/testutils/src/main/resources/wsdl/greeter_control.wsdl
----------------------------------------------------------------------
diff --git a/testutils/src/main/resources/wsdl/greeter_control.wsdl b/testutils/src/main/resources/wsdl/greeter_control.wsdl
index 72ed441..7075c9b 100644
--- a/testutils/src/main/resources/wsdl/greeter_control.wsdl
+++ b/testutils/src/main/resources/wsdl/greeter_control.wsdl
@@ -348,5 +348,8 @@
         <wsdl:port binding="tns:ControlSOAPBinding" name="ReplicatedPortD">
             <soap:address location="http://localhost:9054/SoapContext/ReplicatedPortD"/>
         </wsdl:port>
+        <wsdl:port binding="tns:GreeterSOAPBinding" name="ReplicatedPortE">
+            <soap:address location="http://localhost:9055/SoapContext/ReplicatedPortE"/>
+        </wsdl:port>
     </wsdl:service>
 </wsdl:definitions>