You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stratos.apache.org by ra...@apache.org on 2015/08/30 20:17:02 UTC

[09/50] [abbrv] stratos git commit: Fixing intermittent issue in group termination behavior integration test: adding assertions for member termination

Fixing intermittent issue in group termination behavior integration test: adding assertions for member termination


Project: http://git-wip-us.apache.org/repos/asf/stratos/repo
Commit: http://git-wip-us.apache.org/repos/asf/stratos/commit/d72139dc
Tree: http://git-wip-us.apache.org/repos/asf/stratos/tree/d72139dc
Diff: http://git-wip-us.apache.org/repos/asf/stratos/diff/d72139dc

Branch: refs/heads/master
Commit: d72139dcd303ffbb8de4793444d516410c1dac6d
Parents: 7520d69
Author: Akila Perera <ra...@gmail.com>
Authored: Wed Aug 26 18:15:08 2015 +0530
Committer: Akila Perera <ra...@gmail.com>
Committed: Wed Aug 26 18:15:25 2015 +0530

----------------------------------------------------------------------
 .../mock/iaas/client/MockIaasApiClient.java     |   4 +-
 .../integration/tests/TopologyHandler.java      | 130 +++++++++++++------
 .../GroupTerminationBehaviorTest.java           |  73 +++++++----
 .../tests/rest/IntegrationMockClient.java       |   7 +-
 .../src/test/resources/stratos-testing.xml      |  32 ++---
 5 files changed, 156 insertions(+), 90 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/stratos/blob/d72139dc/components/org.apache.stratos.mock.iaas.client/src/main/java/org/apache/stratos/mock/iaas/client/MockIaasApiClient.java
----------------------------------------------------------------------
diff --git a/components/org.apache.stratos.mock.iaas.client/src/main/java/org/apache/stratos/mock/iaas/client/MockIaasApiClient.java b/components/org.apache.stratos.mock.iaas.client/src/main/java/org/apache/stratos/mock/iaas/client/MockIaasApiClient.java
index b76bc14..47bbd17 100644
--- a/components/org.apache.stratos.mock.iaas.client/src/main/java/org/apache/stratos/mock/iaas/client/MockIaasApiClient.java
+++ b/components/org.apache.stratos.mock.iaas.client/src/main/java/org/apache/stratos/mock/iaas/client/MockIaasApiClient.java
@@ -75,7 +75,7 @@ public class MockIaasApiClient {
         }
     }
 
-    public void terminateInstance(String instanceId) {
+    public boolean terminateInstance(String instanceId) {
         try {
             if (log.isDebugEnabled()) {
                 log.debug(String.format("Terminate instance: [instance-id] %s", instanceId));
@@ -84,7 +84,7 @@ public class MockIaasApiClient {
             HttpResponse response = restClient.doDelete(uri);
             if (response != null) {
                 if ((response.getStatusCode() >= 200) && (response.getStatusCode() < 300)) {
-                    return;
+                    return false;
                 } else {
                     GsonBuilder gsonBuilder = new GsonBuilder();
                     Gson gson = gsonBuilder.create();

http://git-wip-us.apache.org/repos/asf/stratos/blob/d72139dc/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/TopologyHandler.java
----------------------------------------------------------------------
diff --git a/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/TopologyHandler.java b/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/TopologyHandler.java
index 9c9764b..85f7817 100644
--- a/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/TopologyHandler.java
+++ b/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/TopologyHandler.java
@@ -45,6 +45,7 @@ import org.apache.stratos.messaging.message.receiver.topology.TopologyManager;
 import java.io.File;
 import java.rmi.RemoteException;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -60,6 +61,7 @@ public class TopologyHandler {
 
     public static final int APPLICATION_ACTIVATION_TIMEOUT = 360000;
     public static final int APPLICATION_UNDEPLOYMENT_TIMEOUT = 120000;
+    public static final int MEMBER_TERMINATION_TIMEOUT = 120000;
     public static final int APPLICATION_TOPOLOGY_TIMEOUT = 90000;
     public static final String APPLICATION_STATUS_CREATED = "Created";
     public static final String APPLICATION_STATUS_UNDEPLOYING = "Undeploying";
@@ -129,14 +131,16 @@ public class TopologyHandler {
         while (!applicationTopologyInitialized) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             applicationTopologyInitialized = ApplicationManager.getApplications().isInitialized();
             if ((System.currentTimeMillis() - startTime) > APPLICATION_TOPOLOGY_TIMEOUT) {
                 break;
             }
         }
-        assertEquals(String.format("Application Topology didn't get initialized "), applicationTopologyInitialized, true);
+        assertEquals(String.format("Application Topology didn't get initialized "), applicationTopologyInitialized,
+                true);
     }
 
     /**
@@ -148,7 +152,8 @@ public class TopologyHandler {
         while (!topologyInitialized) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             topologyInitialized = TopologyManager.getTopology().isInitialized();
             if ((System.currentTimeMillis() - startTime) > APPLICATION_TOPOLOGY_TIMEOUT) {
@@ -169,7 +174,8 @@ public class TopologyHandler {
         while (!((application != null) && (application.getStatus() == status))) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             application = ApplicationManager.getApplications().getApplication(applicationName);
             if ((System.currentTimeMillis() - startTime) > APPLICATION_ACTIVATION_TIMEOUT) {
@@ -243,20 +249,20 @@ public class TopologyHandler {
     }
 
     /**
-     * Assert application activation
+     * Get all the members that belongs to the cluster identified by cartridge name and application name in the
+     * topology
      *
+     * @param cartridgeName
      * @param applicationName
      */
-    public void terminateMemberFromCluster(String cartridgeName, String applicationName,
-                                           IntegrationMockClient mockIaasApiClient) {
+    public Map<String, Member> getMembersForCluster(String cartridgeName, String applicationName) {
         Application application = ApplicationManager.getApplications().getApplication(applicationName);
-        assertNotNull(String.format("Application is not found: [application-id] %s",
-                applicationName), application);
-
+        assertNotNull(String.format("Application is not found: [application-id] %s", applicationName), application);
         Set<ClusterDataHolder> clusterDataHolderSet = application.getClusterDataRecursively();
+        Map<String, Member> memberMap = new HashMap<String, Member>();
         for (ClusterDataHolder clusterDataHolder : clusterDataHolderSet) {
             String serviceName = clusterDataHolder.getServiceType();
-            if(cartridgeName.equals(serviceName)) {
+            if (cartridgeName.equals(serviceName)) {
                 String clusterId = clusterDataHolder.getClusterId();
                 Service service = TopologyManager.getTopology().getService(serviceName);
                 assertNotNull(String.format("Service is not found: [application-id] %s [service] %s",
@@ -265,29 +271,56 @@ public class TopologyHandler {
                 Cluster cluster = service.getCluster(clusterId);
                 assertNotNull(String.format("Cluster is not found: [application-id] %s [service] %s [cluster-id] %s",
                         applicationName, serviceName, clusterId), cluster);
-                boolean memberTerminated = false;
-
                 for (ClusterInstance instance : cluster.getInstanceIdToInstanceContextMap().values()) {
                     for (Member member : cluster.getMembers()) {
-                        if (member.getClusterInstanceId().equals(instance.getInstanceId())) {
-                            if (member.getStatus().equals(MemberStatus.Active)) {
-                                mockIaasApiClient.terminateInstance(member.getMemberId());
-                                memberTerminated = true;
-                                break;
-                            }
-                        }
+                        memberMap.put(member.getMemberId(), member);
                     }
+                }
+            }
+        }
+        return memberMap;
+    }
 
-                    if(memberTerminated) {
-                        break;
-                    }
+    /**
+     * Terminate a member in mock iaas directly without involving Stratos REST API
+     * This is similar to manually terminating an instance in an IaaS. This could be used to simulate member failures
+     *
+     * @param memberId
+     * @param mockIaasApiClient
+     */
+    public void terminateMemberInMockIaas(String memberId, IntegrationMockClient mockIaasApiClient) {
+        boolean memberTerminated = false;
+        memberTerminated = mockIaasApiClient.terminateInstance(memberId);
+        assertTrue(String.format("Member [member-id] %s couldn't be terminated from the mock IaaS", memberId),
+                memberTerminated);
+    }
 
+    public void assertMemberTermination(String memberId) {
+        long startTime = System.currentTimeMillis();
+        assertNotNull(String.format("Member id is not found: [member-id] %s", memberId));
+        boolean hasMemberRemoved = false;
+        while (!hasMemberRemoved) {
+            // Wait until the member gets removed by MemberTerminatedEvent topology receiver
+            if (getTerminatingMembers().get(memberId) == null &&
+                    getInActiveMembers().get(memberId) == null &&
+                    getActivateddMembers().get(memberId) == null &&
+                    getCreatedMembers().get(memberId) == null) {
+                getTerminatedMembers().remove(memberId);
+                hasMemberRemoved = true;
+            } else {
+                if (getTerminatedMembers().get(memberId) - startTime > MEMBER_TERMINATION_TIMEOUT) {
+                    break;
                 }
-                assertTrue("Any member couldn't be terminated from the mock IaaS client", memberTerminated);
             }
-
+            try {
+                Thread.sleep(1000);
+            }
+            catch (InterruptedException e) {
+                log.error("Could not sleep", e);
+            }
         }
-
+        assertTrue(String.format("Member [member-id] %s did not get removed from the topology", memberId),
+                hasMemberRemoved);
     }
 
     public void assertClusterMinMemberCount(String applicationName, int minMembers) {
@@ -324,7 +357,8 @@ public class TopologyHandler {
                 while (!clusterActive) {
                     try {
                         Thread.sleep(1000);
-                    } catch (InterruptedException ignore) {
+                    }
+                    catch (InterruptedException ignore) {
                     }
                     service = TopologyManager.getTopology().getService(serviceName);
                     assertNotNull(String.format("Service is not found: [application-id] %s [service] %s",
@@ -340,8 +374,9 @@ public class TopologyHandler {
                         }
                     }
                     clusterActive = activeInstances >= minMembers;
-                    assertNotNull(String.format("Cluster is not found: [application-id] %s [service] %s [cluster-id] %s",
-                            applicationName, serviceName, clusterId), cluster);
+                    assertNotNull(
+                            String.format("Cluster is not found: [application-id] %s [service] %s [cluster-id] %s",
+                                    applicationName, serviceName, clusterId), cluster);
 
                     if ((System.currentTimeMillis() - startTime) > APPLICATION_ACTIVATION_TIMEOUT) {
                         break;
@@ -366,19 +401,22 @@ public class TopologyHandler {
         ApplicationContext applicationContext = null;
         try {
             applicationContext = AutoscalerServiceClient.getInstance().getApplication(applicationName);
-        } catch (RemoteException e) {
+        }
+        catch (RemoteException e) {
             log.error("Error while getting the application context for [application] " + applicationName);
         }
         while (((application != null) && application.getInstanceContextCount() > 0) ||
                 (applicationContext == null || applicationContext.getStatus().equals(APPLICATION_STATUS_UNDEPLOYING))) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             application = ApplicationManager.getApplications().getApplication(applicationName);
             try {
                 applicationContext = AutoscalerServiceClient.getInstance().getApplication(applicationName);
-            } catch (RemoteException e) {
+            }
+            catch (RemoteException e) {
                 log.error("Error while getting the application context for [application] " + applicationName);
             }
             if ((System.currentTimeMillis() - startTime) > APPLICATION_UNDEPLOYMENT_TIMEOUT) {
@@ -396,7 +434,8 @@ public class TopologyHandler {
                 applicationContext.getStatus().equals(APPLICATION_STATUS_UNDEPLOYING)) {
             return false;
         }
-        assertEquals(String.format("Application status did not change to Created: [application-id] %s", applicationName),
+        assertEquals(
+                String.format("Application status did not change to Created: [application-id] %s", applicationName),
                 APPLICATION_STATUS_CREATED, applicationContext.getStatus());
         return true;
     }
@@ -414,7 +453,8 @@ public class TopologyHandler {
             while (group.getInstanceContextCount() != count) {
                 try {
                     Thread.sleep(1000);
-                } catch (InterruptedException ignore) {
+                }
+                catch (InterruptedException ignore) {
                 }
                 if ((System.currentTimeMillis() - startTime) > APPLICATION_ACTIVATION_TIMEOUT) {
                     break;
@@ -424,14 +464,16 @@ public class TopologyHandler {
                 while (!instance.getStatus().equals(GroupStatus.Active)) {
                     try {
                         Thread.sleep(1000);
-                    } catch (InterruptedException ignore) {
+                    }
+                    catch (InterruptedException ignore) {
                     }
                     if ((System.currentTimeMillis() - startTime) > APPLICATION_ACTIVATION_TIMEOUT) {
                         break;
                     }
                 }
             }
-            assertEquals(String.format("Application status did not change to active: [application-id] %s", applicationName),
+            assertEquals(
+                    String.format("Application status did not change to active: [application-id] %s", applicationName),
                     group.getInstanceContextCount(), count);
         }
         assertNotNull(String.format("Application is not found: [application-id] %s", applicationName), application);
@@ -440,7 +482,8 @@ public class TopologyHandler {
 
     public void assertApplicationNotExists(String applicationName) {
         Application application = ApplicationManager.getApplications().getApplication(applicationName);
-        assertNull(String.format("Application is found in the topology : [application-id] %s", applicationName), application);
+        assertNull(String.format("Application is found in the topology : [application-id] %s", applicationName),
+                application);
     }
 
     /**
@@ -459,7 +502,10 @@ public class TopologyHandler {
             protected void onEvent(Event event) {
                 MemberTerminatedEvent memberTerminatedEvent = (MemberTerminatedEvent) event;
                 getTerminatedMembers().put(memberTerminatedEvent.getMemberId(), System.currentTimeMillis());
-
+                getActivateddMembers().remove(((MemberTerminatedEvent) event).getMemberId());
+                getCreatedMembers().remove(((MemberTerminatedEvent) event).getMemberId());
+                getInActiveMembers().remove(((MemberTerminatedEvent) event).getMemberId());
+                getTerminatingMembers().remove(((MemberTerminatedEvent) event).getMemberId());
             }
         });
 
@@ -593,14 +639,14 @@ public class TopologyHandler {
 
 
     public void removeMembersFromMaps(String applicationId) {
-        for(Map.Entry<String, Long> entry: getActivateddMembers().entrySet()) {
-            if(entry.getKey().contains(applicationId)) {
+        for (Map.Entry<String, Long> entry : getActivateddMembers().entrySet()) {
+            if (entry.getKey().contains(applicationId)) {
                 getActivateddMembers().remove(entry.getKey());
             }
         }
 
-        for(Map.Entry<String, Long> entry: getTerminatedMembers().entrySet()) {
-            if(entry.getKey().contains(applicationId)) {
+        for (Map.Entry<String, Long> entry : getTerminatedMembers().entrySet()) {
+            if (entry.getKey().contains(applicationId)) {
                 getTerminatedMembers().remove(entry.getKey());
             }
         }

http://git-wip-us.apache.org/repos/asf/stratos/blob/d72139dc/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/application/GroupTerminationBehaviorTest.java
----------------------------------------------------------------------
diff --git a/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/application/GroupTerminationBehaviorTest.java b/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/application/GroupTerminationBehaviorTest.java
index 3cc4ebd..6ea7150 100644
--- a/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/application/GroupTerminationBehaviorTest.java
+++ b/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/application/GroupTerminationBehaviorTest.java
@@ -27,6 +27,7 @@ import org.apache.stratos.integration.tests.RestConstants;
 import org.apache.stratos.integration.tests.StratosTestServerManager;
 import org.apache.stratos.integration.tests.TopologyHandler;
 import org.apache.stratos.messaging.domain.application.ApplicationStatus;
+import org.apache.stratos.messaging.domain.topology.Member;
 import org.testng.annotations.Test;
 
 import java.util.ArrayList;
@@ -46,7 +47,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
     @Test
     public void testTerminationBehavior() {
         try {
-            log.info("-------------------------------Started application termination behavior test case-------------------------------");
+            log.info(
+                    "-------------------------------Started application termination behavior test case-------------------------------");
 
             String autoscalingPolicyId = "autoscaling-policy-group-termination-behavior-test";
             TopologyHandler topologyHandler = TopologyHandler.getInstance();
@@ -56,24 +58,29 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
                     RestConstants.AUTOSCALING_POLICIES, RestConstants.AUTOSCALING_POLICIES_NAME);
             assertTrue(addedScalingPolicy);
 
-            boolean addedC1 = restClient.addEntity(RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c1-group-termination-behavior-test.json",
+            boolean addedC1 = restClient.addEntity(
+                    RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c1-group-termination-behavior-test.json",
                     RestConstants.CARTRIDGES, RestConstants.CARTRIDGES_NAME);
             assertTrue(addedC1);
 
-            boolean addedC2 = restClient.addEntity(RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c2-group-termination-behavior-test.json",
+            boolean addedC2 = restClient.addEntity(
+                    RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c2-group-termination-behavior-test.json",
                     RestConstants.CARTRIDGES, RestConstants.CARTRIDGES_NAME);
             assertTrue(addedC2);
 
-            boolean addedC3 = restClient.addEntity(RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c3-group-termination-behavior-test.json",
+            boolean addedC3 = restClient.addEntity(
+                    RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c3-group-termination-behavior-test.json",
                     RestConstants.CARTRIDGES, RestConstants.CARTRIDGES_NAME);
             assertTrue(addedC3);
 
-            boolean addedC4 = restClient.addEntity(RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c4-group-termination-behavior-test.json",
+            boolean addedC4 = restClient.addEntity(
+                    RESOURCES_PATH + RestConstants.CARTRIDGES_PATH + "/" + "c4-group-termination-behavior-test.json",
                     RestConstants.CARTRIDGES, RestConstants.CARTRIDGES_NAME);
             assertTrue(addedC4);
 
             boolean addedG1 = restClient.addEntity(RESOURCES_PATH + RestConstants.CARTRIDGE_GROUPS_PATH +
-                            "/" + "cartridge-groups-group-termination-behavior-test.json", RestConstants.CARTRIDGE_GROUPS,
+                            "/" + "cartridge-groups-group-termination-behavior-test.json",
+                    RestConstants.CARTRIDGE_GROUPS,
                     RestConstants.CARTRIDGE_GROUPS_NAME);
             assertTrue(addedG1);
 
@@ -102,7 +109,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
             assertEquals(bean.getApplicationId(), "group-termination-behavior-test");
 
             boolean addAppPolicy = restClient.addEntity(RESOURCES_PATH + RestConstants.APPLICATION_POLICIES_PATH + "/" +
-                            "application-policy-group-termination-behavior-test.json", RestConstants.APPLICATION_POLICIES,
+                            "application-policy-group-termination-behavior-test.json",
+                    RestConstants.APPLICATION_POLICIES,
                     RestConstants.APPLICATION_POLICIES_NAME);
             assertTrue(addAppPolicy);
 
@@ -147,11 +155,15 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
             //Cluster active handling
             topologyHandler.assertClusterActivation(bean.getApplicationId());
 
-            //Terminate one member in the cluster
-            TopologyHandler.getInstance().terminateMemberFromCluster(
-                    "c3-group-termination-behavior-test",
-                    bean.getApplicationId(),
-                    mockIaasApiClient);
+            Map<String, Member> memberMap = TopologyHandler.getInstance().getMembersForCluster
+                    ("c3-group-termination-behavior-test", bean.getApplicationId());
+
+            //Terminate members in the cluster
+            for (Map.Entry<String, Member> entry : memberMap.entrySet()) {
+                String memberId = entry.getValue().getMemberId();
+                TopologyHandler.getInstance().terminateMemberInMockIaas(memberId, mockIaasApiClient);
+                TopologyHandler.getInstance().assertMemberTermination(memberId);
+            }
 
             List<String> clusterIds = new ArrayList<String>();
             clusterIds.add(clusterIdC3);
@@ -208,7 +220,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
             boolean undeploy = topologyHandler.assertApplicationUndeploy("group-termination-behavior-test");
             if (!undeploy) {
                 //Need to forcefully undeploy the application
-                log.info("Force undeployment is going to start for the [application] " + "group-termination-behavior-test");
+                log.info("Force undeployment is going to start for the [application] " +
+                        "group-termination-behavior-test");
 
                 restClient.undeployEntity(RestConstants.APPLICATIONS + "/" + "group-termination-behavior-test" +
                         RestConstants.APPLICATIONS_UNDEPLOY + "?force=true", RestConstants.APPLICATIONS);
@@ -268,9 +281,11 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
                     "network-partition-group-termination-behavior-test-1", RestConstants.NETWORK_PARTITIONS_NAME);
             assertTrue(removedNet);
 
-            log.info("-------------------------------Ended application termination behavior test case-------------------------------");
+            log.info(
+                    "-------------------------------Ended application termination behavior test case-------------------------------");
 
-        } catch (Exception e) {
+        }
+        catch (Exception e) {
             log.error("An error occurred while handling  application termination behavior", e);
             assertTrue("An error occurred while handling  application termination behavior", false);
         }
@@ -283,7 +298,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
         while (!inActiveMap.containsKey(clusterId)) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             inActiveMap = TopologyHandler.getInstance().getInActiveMembers();
             if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {
@@ -295,7 +311,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
         while (!inActiveMap.containsKey(groupId)) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             inActiveMap = TopologyHandler.getInstance().getInActiveMembers();
             if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {
@@ -313,7 +330,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
             while (!terminatingMembers.containsKey(clusterId)) {
                 try {
                     Thread.sleep(1000);
-                } catch (InterruptedException ignore) {
+                }
+                catch (InterruptedException ignore) {
                 }
                 terminatingMembers = TopologyHandler.getInstance().getTerminatingMembers();
                 if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {
@@ -326,7 +344,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
         while (!terminatingMembers.containsKey(groupId)) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             terminatingMembers = TopologyHandler.getInstance().getTerminatingMembers();
             if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {
@@ -345,7 +364,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
             while (!terminatedMembers.containsKey(clusterId)) {
                 try {
                     Thread.sleep(1000);
-                } catch (InterruptedException ignore) {
+                }
+                catch (InterruptedException ignore) {
                 }
                 terminatedMembers = TopologyHandler.getInstance().getTerminatedMembers();
                 if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {
@@ -358,7 +378,8 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
         while (!terminatedMembers.containsKey(groupId)) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException ignore) {
+            }
+            catch (InterruptedException ignore) {
             }
             terminatedMembers = TopologyHandler.getInstance().getTerminatedMembers();
             if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {
@@ -375,10 +396,11 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
         Map<String, Long> activeMembers = TopologyHandler.getInstance().getActivateddMembers();
         Map<String, Long> createdMembers = TopologyHandler.getInstance().getCreatedMembers();
         //Active member should be available at the time cluster is started to create.
-        while(!activeMembers.containsKey(firstNodeId)) {
+        while (!activeMembers.containsKey(firstNodeId)) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException e) {
+            }
+            catch (InterruptedException e) {
             }
             activeMembers = TopologyHandler.getInstance().getActivateddMembers();
             if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {
@@ -387,10 +409,11 @@ public class GroupTerminationBehaviorTest extends StratosTestServerManager {
         }
         assertTrue(activeMembers.containsKey(firstNodeId));
 
-        while(!createdMembers.containsKey(secondNodeId)) {
+        while (!createdMembers.containsKey(secondNodeId)) {
             try {
                 Thread.sleep(1000);
-            } catch (InterruptedException e) {
+            }
+            catch (InterruptedException e) {
             }
             createdMembers = TopologyHandler.getInstance().getCreatedMembers();
             if ((System.currentTimeMillis() - startTime) > GROUP_INACTIVE_TIMEOUT) {

http://git-wip-us.apache.org/repos/asf/stratos/blob/d72139dc/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/rest/IntegrationMockClient.java
----------------------------------------------------------------------
diff --git a/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/rest/IntegrationMockClient.java b/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/rest/IntegrationMockClient.java
index dbe21cb..0378509 100644
--- a/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/rest/IntegrationMockClient.java
+++ b/products/stratos/modules/integration/src/test/java/org/apache/stratos/integration/tests/rest/IntegrationMockClient.java
@@ -28,11 +28,8 @@ import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.impl.conn.PoolingClientConnectionManager;
 import org.apache.stratos.mock.iaas.client.MockIaasApiClient;
-import org.apache.stratos.mock.iaas.client.rest.*;
 import org.apache.stratos.mock.iaas.client.rest.HttpResponse;
 import org.apache.stratos.mock.iaas.client.rest.HttpResponseHandler;
-import org.apache.stratos.mock.iaas.client.rest.RestClient;
-import org.apache.stratos.mock.iaas.domain.*;
 
 import java.net.URI;
 
@@ -58,7 +55,7 @@ public class IntegrationMockClient extends MockIaasApiClient {
         httpClient = (DefaultHttpClient) WebClientWrapper.wrapClient(httpClient);
     }
 
-    public void terminateInstance(String instanceId) {
+    public boolean terminateInstance(String instanceId) {
         try {
             if (log.isDebugEnabled()) {
                 log.debug(String.format("Terminate instance: [instance-id] %s", instanceId));
@@ -67,7 +64,7 @@ public class IntegrationMockClient extends MockIaasApiClient {
             org.apache.stratos.mock.iaas.client.rest.HttpResponse response = doDelete(uri);
             if (response != null) {
                 if ((response.getStatusCode() >= 200) && (response.getStatusCode() < 300)) {
-                    return;
+                    return true;
                 } else {
                     GsonBuilder gsonBuilder = new GsonBuilder();
                     Gson gson = gsonBuilder.create();

http://git-wip-us.apache.org/repos/asf/stratos/blob/d72139dc/products/stratos/modules/integration/src/test/resources/stratos-testing.xml
----------------------------------------------------------------------
diff --git a/products/stratos/modules/integration/src/test/resources/stratos-testing.xml b/products/stratos/modules/integration/src/test/resources/stratos-testing.xml
index e8a48d2..2e0d2a8 100644
--- a/products/stratos/modules/integration/src/test/resources/stratos-testing.xml
+++ b/products/stratos/modules/integration/src/test/resources/stratos-testing.xml
@@ -23,72 +23,72 @@
 <suite name="StratosIntegrationSuite">
     <test name="UserTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.users.UserTest" />
+            <class name="org.apache.stratos.integration.tests.users.UserTest"/>
         </classes>
     </test>
     <test name="CartridgeTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.group.CartridgeTest" />
+            <class name="org.apache.stratos.integration.tests.group.CartridgeTest"/>
         </classes>
     </test>
-    <test name="CartridgeGroupTest" >
+    <test name="CartridgeGroupTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.group.CartridgeGroupTest" />
+            <class name="org.apache.stratos.integration.tests.group.CartridgeGroupTest"/>
         </classes>
     </test>
     <test name="NetworkPartitionTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.policies.NetworkPartitionTest" />
+            <class name="org.apache.stratos.integration.tests.policies.NetworkPartitionTest"/>
         </classes>
     </test>
     <test name="ApplicationPolicyTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.policies.ApplicationPolicyTest" />
+            <class name="org.apache.stratos.integration.tests.policies.ApplicationPolicyTest"/>
         </classes>
     </test>
     <test name="DeploymentPolicyTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.policies.DeploymentPolicyTest" />
+            <class name="org.apache.stratos.integration.tests.policies.DeploymentPolicyTest"/>
         </classes>
     </test>
     <test name="AutoscalingPolicyTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.policies.AutoscalingPolicyTest" />
+            <class name="org.apache.stratos.integration.tests.policies.AutoscalingPolicyTest"/>
         </classes>
     </test>
     <test name="SampleApplicationsTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.application.SampleApplicationsTest" />
+            <class name="org.apache.stratos.integration.tests.application.SampleApplicationsTest"/>
         </classes>
     </test>
     <test name="ApplicationUpdateTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.application.ApplicationUpdateTest" />
+            <class name="org.apache.stratos.integration.tests.application.ApplicationUpdateTest"/>
         </classes>
     </test>
     <test name="SingleClusterScalingTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.application.SingleClusterScalingTest" />
+            <class name="org.apache.stratos.integration.tests.application.SingleClusterScalingTest"/>
         </classes>
     </test>
     <test name="ApplicationBurstingTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.application.ApplicationBurstingTest" />
+            <class name="org.apache.stratos.integration.tests.application.ApplicationBurstingTest"/>
         </classes>
     </test>
     <test name="PartitionRoundRobinClusterTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.application.PartitionRoundRobinClusterTest" />
+            <class name="org.apache.stratos.integration.tests.application.PartitionRoundRobinClusterTest"/>
         </classes>
     </test>
     <test name="GroupStartupOrderTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.application.GroupStartupOrderTest" />
+            <class name="org.apache.stratos.integration.tests.application.GroupStartupOrderTest"/>
         </classes>
     </test>
     <test name="GroupTerminationBehaviorTest">
         <classes>
-            <class name="org.apache.stratos.integration.tests.application.GroupTerminationBehaviorTest" />
+            <class name="org.apache.stratos.integration.tests.application.GroupTerminationBehaviorTest"/>
         </classes>
     </test>
-</suite>
+</suite>
\ No newline at end of file