You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by da...@apache.org on 2014/07/28 13:59:10 UTC

[1/2] CLOUDSTACK-6445: Simulator enhancements Refer FS - https://cwiki.apache.org/confluence/display/CLOUDSTACK/Simulator+enhancements

Repository: cloudstack
Updated Branches:
  refs/heads/4.4 19eb06bd3 -> 25b415972


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/test/integration/smoke/test_vm_ha.py
----------------------------------------------------------------------
diff --git a/test/integration/smoke/test_vm_ha.py b/test/integration/smoke/test_vm_ha.py
new file mode 100644
index 0000000..ae5b51f
--- /dev/null
+++ b/test/integration/smoke/test_vm_ha.py
@@ -0,0 +1,232 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#Test from the Marvin - Testing in Python wiki
+
+import time
+
+#All tests inherit from cloudstackTestCase
+from marvin.cloudstackTestCase import cloudstackTestCase
+
+#Import Integration Libraries
+
+#base - contains all resources as entities and defines create, delete, list operations on them
+from marvin.integration.lib.base import Account, VirtualMachine, Cluster, Host, ServiceOffering, Configurations, SimulatorMock
+
+#utils - utility classes for common cleanup, external library wrappers etc
+from marvin.integration.lib.utils import cleanup_resources
+
+#common - commonly used methods for all tests are listed here
+from marvin.integration.lib.common import get_zone, get_domain, get_template
+
+from nose.plugins.attrib import attr
+
+class TestData(object):
+    """Test data object that is required to create resources
+    """
+    def __init__(self):
+        self.testdata = {
+            #data to create an account
+            "account": {
+                "email": "test@test.com",
+                "firstname": "Test",
+                "lastname": "User",
+                "username": "test",
+                "password": "password",
+            },
+            #data reqd for virtual machine creation
+            "virtual_machine" : {
+                "name" : "testvm",
+                "displayname" : "Test VM",
+            },
+            #small service offering
+            "service_offering": {
+                "hasmall": {
+                    "name": "HA Small Instance",
+                    "displaytext": "HA Small Instance",
+                    "cpunumber": 1,
+                    "cpuspeed": 100,
+                    "memory": 256,
+                    "hosttags": "ha",                    
+                    "offerha": True,
+                },
+            },
+            "ostype": 'CentOS 5.3 (32-bit)',
+        }
+
+
+class TestDeployVMHA(cloudstackTestCase):
+    """Test VM HA
+    """
+
+    def setUp(self):
+        self.testdata = TestData().testdata
+        self.apiclient = self.testClient.getApiClient()
+
+        # Get Zone, Domain and Default Built-in template
+        self.domain = get_domain(self.apiclient, self.testdata)
+        self.zone = get_zone(self.apiclient, self.testdata)
+        self.testdata["mode"] = self.zone.networktype
+        self.template = get_template(self.apiclient, self.zone.id, self.testdata["ostype"])
+
+        self.hosts = []
+        suitablecluster = None
+        clusters = Cluster.list(self.apiclient)
+        self.assertTrue(isinstance(clusters, list) and len(clusters) > 0, msg = "No clusters found")
+        for cluster in clusters:
+            self.hosts = Host.list(self.apiclient, clusterid=cluster.id, type='Routing')
+            if isinstance(self.hosts, list) and len(self.hosts) >= 2:
+                suitablecluster = cluster
+                break
+        self.assertTrue(isinstance(self.hosts, list) and len(self.hosts) >= 2, msg = "Atleast 2 hosts required in cluster for VM HA test")
+        #update host tags
+        for host in self.hosts:
+            Host.update(self.apiclient, id=host.id, hosttags=self.testdata["service_offering"]["hasmall"]["hosttags"])
+
+        #create a user account
+        self.account = Account.create(
+            self.apiclient,
+            self.testdata["account"],
+            domainid=self.domain.id
+        )
+        #create a service offering
+        self.service_offering = ServiceOffering.create(
+            self.apiclient,
+            self.testdata["service_offering"]["hasmall"]
+        )
+        #deploy ha vm
+        self.virtual_machine = VirtualMachine.create(
+            self.apiclient,
+            self.testdata["virtual_machine"],
+            accountid=self.account.name,
+            zoneid=self.zone.id,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering.id,
+            templateid=self.template.id
+        )
+        list_vms = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)
+        self.debug(
+            "Verify listVirtualMachines response for virtual machine: %s"\
+            % self.virtual_machine.id
+        )
+        self.assertTrue(isinstance(list_vms, list) and len(list_vms) == 1, msg = "List VM response was empty")
+        self.virtual_machine = list_vms[0]
+
+        self.mock_checkhealth = SimulatorMock.create(
+            apiclient=self.apiclient,
+            command="CheckHealthCommand",
+            zoneid=suitablecluster.zoneid,
+            podid=suitablecluster.podid,
+            clusterid=suitablecluster.id,
+            hostid=self.virtual_machine.hostid,
+            value="result:fail")
+        self.mock_ping = SimulatorMock.create(
+            apiclient=self.apiclient,
+            command="PingCommand",
+            zoneid=suitablecluster.zoneid,
+            podid=suitablecluster.podid,
+            clusterid=suitablecluster.id,
+            hostid=self.virtual_machine.hostid,
+            value="result:fail")
+        self.mock_checkvirtualmachine = SimulatorMock.create(
+            apiclient=self.apiclient,
+            command="CheckVirtualMachineCommand",
+            zoneid=suitablecluster.zoneid,
+            podid=suitablecluster.podid,
+            clusterid=suitablecluster.id,
+            hostid=self.virtual_machine.hostid,
+            value="result:fail")
+        self.mock_pingtest = SimulatorMock.create(
+            apiclient=self.apiclient,
+            command="PingTestCommand",
+            zoneid=suitablecluster.zoneid,
+            podid=suitablecluster.podid,
+            value="result:fail")
+        self.mock_checkonhost_list = []
+        for host in self.hosts:
+            if host.id != self.virtual_machine.hostid:
+                self.mock_checkonhost_list.append(SimulatorMock.create(
+                    apiclient=self.apiclient,
+                    command="CheckOnHostCommand",
+                    zoneid=suitablecluster.zoneid,
+                    podid=suitablecluster.podid,
+                    clusterid=suitablecluster.id,
+                    hostid=host.id,
+                    value="result:fail"))
+        #build cleanup list
+        self.cleanup = [
+            self.service_offering,
+            self.account,
+            self.mock_checkhealth,
+            self.mock_ping,
+            self.mock_checkvirtualmachine,
+            self.mock_pingtest
+        ]
+        self.cleanup = self.cleanup + self.mock_checkonhost_list
+
+    @attr(tags = ['selfservice'])
+    def test_vm_ha(self):
+        """Test VM HA
+
+        # Validate the following:
+        # VM started on other host in cluster
+        """
+
+        #wait for VM to HA
+        ping_timeout = Configurations.list(self.apiclient, name="ping.timeout")
+        ping_interval = Configurations.list(self.apiclient, name="ping.interval")
+        total_duration = int(float(ping_timeout[0].value) * float(ping_interval[0].value))
+        time.sleep(total_duration)
+
+        duration = 0
+        vm = None
+        while duration < total_duration:
+            list_vms = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)
+            self.assertTrue(isinstance(list_vms, list) and len(list_vms) == 1, msg = "List VM response was empty")
+            vm = list_vms[0]
+            if vm.hostid != self.virtual_machine.hostid:
+                break
+            else:
+                time.sleep(10)
+                duration = duration + 10
+
+        self.assertEqual(
+            vm.id,
+            self.virtual_machine.id,
+            "VM ids do not match")
+        self.assertEqual(
+            vm.name,
+            self.virtual_machine.name,
+            "VM names do not match")
+        self.assertEqual(
+            vm.state,
+            "Running",
+            msg="VM is not in Running state")
+        self.assertNotEqual(
+            vm.hostid,
+            self.virtual_machine.hostid,
+            msg="VM is not started on another host as part of HA")
+
+    def tearDown(self):
+        try:
+            for host in self.hosts:
+                Host.update(self.apiclient, id=host.id, hosttags="")
+
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            self.debug("Warning! Exception in tearDown: %s" % e)
+

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/tools/marvin/marvin/integration/lib/base.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/integration/lib/base.py b/tools/marvin/marvin/integration/lib/base.py
index 95b7fe9..b0f7f8c 100755
--- a/tools/marvin/marvin/integration/lib/base.py
+++ b/tools/marvin/marvin/integration/lib/base.py
@@ -1462,6 +1462,9 @@ class ServiceOffering:
         if "tags" in services:
             cmd.tags = services["tags"]
 
+        if "hosttags" in services:
+            cmd.hosttags = services["hosttags"]
+
         if "deploymentplanner" in services:
             cmd.deploymentplanner = services["deploymentplanner"]
 
@@ -1471,6 +1474,9 @@ class ServiceOffering:
         if "isvolatile" in services:
             cmd.isvolatile = services["isvolatile"]
 
+        if "offerha" in services:
+            cmd.offerha = services["offerha"]
+
         # Service Offering private to that domain
         if domainid:
             cmd.domainid = domainid
@@ -3834,4 +3840,50 @@ class IAMPolicy:
         cmd.id = self.id
         cmd.accounts = [str(acct.id) for acct in accts]
         apiclient.removeIAMPolicyFromAccount(cmd)
-        return           
+        return
+
+class SimulatorMock:
+    """Manage simulator mock lifecycle"""
+    def __init__(self, items):
+        self.__dict__.update(items)
+    
+    @classmethod
+    def create(cls, apiclient, command, zoneid=None, podid=None, clusterid=None, hostid=None, value="result:fail", count=None, jsonresponse=None):
+        """Creates simulator mock"""
+        
+        cmd = configureSimulator.configureSimulatorCmd()
+        cmd.zoneid = zoneid
+        cmd.podid = podid
+        cmd.clusterid = clusterid
+        cmd.hostid = hostid
+        cmd.name = command
+        cmd.value = value
+        cmd.count = count
+        cmd.jsonresponse = jsonresponse
+        try:
+            simulatormock = apiclient.configureSimulator(cmd)
+            if simulatormock is not None:
+                return SimulatorMock(simulatormock.__dict__)
+        except Exception as e:
+            raise e
+    
+    def delete(self, apiclient):
+        """Removes simulator mock"""
+        
+        cmd = cleanupSimulatorMock.cleanupSimulatorMockCmd()
+        cmd.id = self.id
+        return apiclient.cleanupSimulatorMock(cmd)
+    
+    def query(self, apiclient):
+        """Queries simulator mock"""
+        
+        cmd = querySimulatorMock.querySimulatorMockCmd()
+        cmd.id = self.id
+        try:
+            simulatormock = apiclient.querySimulatorMock(cmd)
+            if simulatormock is not None:
+                return SimulatorMock(simulatormock.__dict__)
+        except Exception as e:
+            raise e
+
+


[2/2] git commit: updated refs/heads/4.4 to 25b4159

Posted by da...@apache.org.
CLOUDSTACK-6445: Simulator enhancements
Refer FS - https://cwiki.apache.org/confluence/display/CLOUDSTACK/Simulator+enhancements

(cherry picked from commit 617826d16b4d5220bb3b51ed511b3c065d0e8926)


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/25b41597
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/25b41597
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/25b41597

Branch: refs/heads/4.4
Commit: 25b4159723fc6e3c169f2fcaf009e579fa4e5874
Parents: 19eb06b
Author: Koushik Das <ko...@apache.org>
Authored: Fri Apr 18 11:16:00 2014 +0530
Committer: Daan Hoogland <da...@onecht.net>
Committed: Mon Jul 28 13:58:55 2014 +0200

----------------------------------------------------------------------
 .../tomcatconf/commands-simulator.properties.in |   2 +
 client/tomcatconf/commands.properties.in        |   2 +
 .../spring-simulator-compute-context.xml        |   7 +-
 .../com/cloud/agent/manager/MockVmManager.java  |   3 +
 .../cloud/agent/manager/MockVmManagerImpl.java  |   7 +
 .../cloud/agent/manager/SimulatorManager.java   |  10 +-
 .../agent/manager/SimulatorManagerImpl.java     | 421 ++++++++++++-------
 .../api/commands/CleanupSimulatorMockCmd.java   |  69 +++
 .../api/commands/ConfigureSimulatorCmd.java     |  43 +-
 .../api/commands/QuerySimulatorMockCmd.java     |  78 ++++
 .../com/cloud/api/response/MockResponse.java    | 107 +++++
 .../src/com/cloud/ha/SimulatorFencer.java       | 116 +++++
 .../src/com/cloud/ha/SimulatorInvestigator.java | 102 +++++
 .../cloud/resource/AgentRoutingResource.java    |  24 ++
 .../com/cloud/resource/SimulatorDiscoverer.java |   0
 .../cloud/simulator/MockConfigurationVO.java    |  33 ++
 .../simulator/dao/MockConfigurationDaoImpl.java |   4 +-
 setup/db/create-schema-simulator.sql            |   5 +-
 test/integration/smoke/test_deploy_vm.py        | 242 ++++++++++-
 test/integration/smoke/test_vm_ha.py            | 232 ++++++++++
 tools/marvin/marvin/integration/lib/base.py     |  54 ++-
 21 files changed, 1373 insertions(+), 188 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/agent-simulator/tomcatconf/commands-simulator.properties.in
----------------------------------------------------------------------
diff --git a/agent-simulator/tomcatconf/commands-simulator.properties.in b/agent-simulator/tomcatconf/commands-simulator.properties.in
index ba19e33..4350bb7 100644
--- a/agent-simulator/tomcatconf/commands-simulator.properties.in
+++ b/agent-simulator/tomcatconf/commands-simulator.properties.in
@@ -17,3 +17,5 @@
 
 
 configureSimulator=com.cloud.api.commands.ConfigureSimulatorCmd;1
+querySimulatorMock=com.cloud.api.commands.QuerySimulatorMockCmd;1
+cleanupSimulatorMock=com.cloud.api.commands.CleanupSimulatorMockCmd;1

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/client/tomcatconf/commands.properties.in
----------------------------------------------------------------------
diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in
index 13e08be..d247aa0 100644
--- a/client/tomcatconf/commands.properties.in
+++ b/client/tomcatconf/commands.properties.in
@@ -620,6 +620,8 @@ deleteStratoshereSsp=1
 #### host simulator commands
 
 configureSimulator=1
+querySimulatorMock=1
+cleanupSimulatorMock=1
 
 #### api discovery commands
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/resources/META-INF/cloudstack/simulator-compute/spring-simulator-compute-context.xml
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/resources/META-INF/cloudstack/simulator-compute/spring-simulator-compute-context.xml b/plugins/hypervisors/simulator/resources/META-INF/cloudstack/simulator-compute/spring-simulator-compute-context.xml
index 1146233..ba7581d 100644
--- a/plugins/hypervisors/simulator/resources/META-INF/cloudstack/simulator-compute/spring-simulator-compute-context.xml
+++ b/plugins/hypervisors/simulator/resources/META-INF/cloudstack/simulator-compute/spring-simulator-compute-context.xml
@@ -30,5 +30,10 @@
     <bean id="SimulatorGuru" class="com.cloud.simulator.SimulatorGuru">
         <property name="name" value="Simulator Guru"/>
     </bean>
-
+    <bean id="SimulatorInvestigator" class="com.cloud.ha.SimulatorInvestigator">
+        <property name="name" value="Simulator Investigator"/>
+    </bean>
+    <bean id="SimulatorFencer" class="com.cloud.ha.SimulatorFencer">
+        <property name="name" value="Simulator Fencer"/>
+    </bean>
 </beans>

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManager.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManager.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManager.java
index bc6cdd7..2032c7a 100644
--- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManager.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManager.java
@@ -27,6 +27,7 @@ import com.cloud.agent.api.CheckVirtualMachineCommand;
 import com.cloud.agent.api.CleanupNetworkRulesCmd;
 import com.cloud.agent.api.CreateVMSnapshotCommand;
 import com.cloud.agent.api.DeleteVMSnapshotCommand;
+import com.cloud.agent.api.FenceCommand;
 import com.cloud.agent.api.GetDomRVersionAnswer;
 import com.cloud.agent.api.GetDomRVersionCmd;
 import com.cloud.agent.api.GetVmStatsCommand;
@@ -110,4 +111,6 @@ public interface MockVmManager extends Manager {
     Answer deleteVmSnapshot(DeleteVMSnapshotCommand cmd);
 
     Answer revertVmSnapshot(RevertToVMSnapshotCommand cmd);
+
+    Answer fence(FenceCommand cmd);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManagerImpl.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManagerImpl.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManagerImpl.java
index e0a7e62..ab583e2 100644
--- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManagerImpl.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/MockVmManagerImpl.java
@@ -27,6 +27,7 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
 import com.cloud.network.VirtualNetworkApplianceService;
+
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -41,6 +42,8 @@ import com.cloud.agent.api.CreateVMSnapshotAnswer;
 import com.cloud.agent.api.CreateVMSnapshotCommand;
 import com.cloud.agent.api.DeleteVMSnapshotAnswer;
 import com.cloud.agent.api.DeleteVMSnapshotCommand;
+import com.cloud.agent.api.FenceAnswer;
+import com.cloud.agent.api.FenceCommand;
 import com.cloud.agent.api.GetDomRVersionAnswer;
 import com.cloud.agent.api.GetDomRVersionCmd;
 import com.cloud.agent.api.GetVmStatsAnswer;
@@ -661,4 +664,8 @@ public class MockVmManagerImpl extends ManagerBase implements MockVmManager {
         return maps;
     }
 
+    @Override
+    public Answer fence(FenceCommand cmd) {
+       return new FenceAnswer(cmd);
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManager.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManager.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManager.java
index 4685cd6..485693a 100644
--- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManager.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManager.java
@@ -22,7 +22,9 @@ import java.util.Map;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.StoragePoolInfo;
+import com.cloud.simulator.MockConfigurationVO;
 import com.cloud.simulator.MockVMVO;
+import com.cloud.simulator.dao.MockConfigurationDao;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.Manager;
 import com.cloud.vm.VirtualMachine.State;
@@ -56,11 +58,17 @@ public interface SimulatorManager extends Manager {
 
     StoragePoolInfo getLocalStorage(String hostGuid);
 
-    boolean configureSimulator(Long zoneId, Long podId, Long clusterId, Long hostId, String command, String values);
+    Long configureSimulator(Long zoneId, Long podId, Long clusterId, Long hostId, String command, String values, Integer count, String jsonResponse);
 
     public HashMap<String, Pair<Long, Long>> syncNetworkGroups(String hostGuid);
 
     Map<String, State> getVmStates(String hostGuid);
 
     Map<String, MockVMVO> getVms(String hostGuid);
+
+    MockConfigurationVO querySimulatorMock(Long id);
+
+    boolean clearSimulatorMock(Long id);
+
+    MockConfigurationDao getMockConfigurationDao();
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java
index 0bb0c5c..3bc689e 100644
--- a/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/agent/manager/SimulatorManagerImpl.java
@@ -17,6 +17,7 @@
 package com.cloud.agent.manager;
 
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -108,7 +109,9 @@ import com.cloud.agent.api.storage.DestroyCommand;
 import com.cloud.agent.api.storage.ListTemplateCommand;
 import com.cloud.agent.api.storage.ListVolumeCommand;
 import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand;
+import com.cloud.api.commands.CleanupSimulatorMockCmd;
 import com.cloud.api.commands.ConfigureSimulatorCmd;
+import com.cloud.api.commands.QuerySimulatorMockCmd;
 import com.cloud.resource.SimulatorStorageProcessor;
 import com.cloud.simulator.MockConfigurationVO;
 import com.cloud.simulator.MockHost;
@@ -124,6 +127,7 @@ import com.cloud.utils.db.DB;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.VirtualMachine.State;
+import com.google.gson.Gson;
 
 @Component
 @Local(value = {SimulatorManager.class})
@@ -184,12 +188,16 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage
     public List<Class<?>> getCommands() {
         List<Class<?>> cmdList = new ArrayList<Class<?>>();
         cmdList.add(ConfigureSimulatorCmd.class);
+        cmdList.add(QuerySimulatorMockCmd.class);
+        cmdList.add(CleanupSimulatorMockCmd.class);
         return cmdList;
     }
 
     @DB
     @Override
     public Answer simulate(Command cmd, String hostGuid) {
+        Answer answer = null;
+        Exception exception = null;
         TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
         try {
             MockHost host = _mockHost.findByGuid(hostGuid);
@@ -198,12 +206,12 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage
             if (index != -1) {
                 cmdName = cmdName.substring(index + 1);
             }
-            MockConfigurationVO config = _mockConfigDao.findByNameBottomUP(host.getDataCenterId(), host.getPodId(), host.getClusterId(), host.getId(), cmdName);
 
             SimulatorInfo info = new SimulatorInfo();
             info.setHostUuid(hostGuid);
 
-            if (config != null) {
+            MockConfigurationVO config = _mockConfigDao.findByNameBottomUP(host.getDataCenterId(), host.getPodId(), host.getClusterId(), host.getId(), cmdName);
+            if (config != null && (config.getCount() == null || config.getCount().intValue() > 0)) {
                 Map<String, String> configParameters = config.getParameters();
                 for (Map.Entry<String, String> entry : configParameters.entrySet()) {
                     if (entry.getKey().equalsIgnoreCase("enabled")) {
@@ -214,171 +222,215 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage
                         } catch (NumberFormatException e) {
                             s_logger.debug("invalid timeout parameter: " + e.toString());
                         }
-                    } else if (entry.getKey().equalsIgnoreCase("wait")) {
+                    }
+
+                    if (entry.getKey().equalsIgnoreCase("wait")) {
                         try {
                             int wait = Integer.valueOf(entry.getValue());
                             Thread.sleep(wait);
                         } catch (NumberFormatException e) {
-                            s_logger.debug("invalid timeout parameter: " + e.toString());
+                            s_logger.debug("invalid wait parameter: " + e.toString());
                         } catch (InterruptedException e) {
                             s_logger.debug("thread is interrupted: " + e.toString());
                         }
                     }
+
+                    if (entry.getKey().equalsIgnoreCase("result")) {
+                        String value = entry.getValue();
+                        if (value.equalsIgnoreCase("fail")) {
+                            answer = new Answer(cmd, false, "Simulated failure");
+                        } else if (value.equalsIgnoreCase("fault")) {
+                            exception = new Exception("Simulated fault");
+                        }
+                    }
+                }
+
+                if (exception != null) {
+                    throw exception;
+                }
+
+                if (answer == null) {
+                    String message = config.getJsonResponse();
+                    if (message != null) {
+                        // json response looks like {"<AnswerType>":....}
+                        String answerType = message.split(":")[0].substring(1).replace("\"", "");
+                        if (answerType != null) {
+                            Class<?> clz = null;
+                            try {
+                                clz = Class.forName(answerType);
+                            } catch (ClassNotFoundException e) {
+                            }
+                            if (clz != null) {
+                                answer = (Answer)new Gson().fromJson(message, clz);
+                            }
+                        }
+                    }
                 }
             }
 
-            if (cmd instanceof GetHostStatsCommand) {
-                return _mockAgentMgr.getHostStatistic((GetHostStatsCommand)cmd);
-            } else if (cmd instanceof CheckHealthCommand) {
-                return _mockAgentMgr.checkHealth((CheckHealthCommand)cmd);
-            } else if (cmd instanceof PingTestCommand) {
-                return _mockAgentMgr.pingTest((PingTestCommand)cmd);
-            } else if (cmd instanceof PrepareForMigrationCommand) {
-                return _mockVmMgr.prepareForMigrate((PrepareForMigrationCommand)cmd);
-            } else if (cmd instanceof MigrateCommand) {
-                return _mockVmMgr.Migrate((MigrateCommand)cmd, info);
-            } else if (cmd instanceof StartCommand) {
-                return _mockVmMgr.startVM((StartCommand)cmd, info);
-            } else if (cmd instanceof CheckSshCommand) {
-                return _mockVmMgr.checkSshCommand((CheckSshCommand)cmd);
-            } else if (cmd instanceof CheckVirtualMachineCommand) {
-                return _mockVmMgr.checkVmState((CheckVirtualMachineCommand)cmd);
-            } else if (cmd instanceof SetStaticNatRulesCommand) {
-                return _mockNetworkMgr.SetStaticNatRules((SetStaticNatRulesCommand)cmd);
-            } else if (cmd instanceof SetFirewallRulesCommand) {
-                return _mockNetworkMgr.SetFirewallRules((SetFirewallRulesCommand)cmd);
-            } else if (cmd instanceof SetPortForwardingRulesCommand) {
-                return _mockNetworkMgr.SetPortForwardingRules((SetPortForwardingRulesCommand)cmd);
-            } else if (cmd instanceof NetworkUsageCommand) {
-                return _mockNetworkMgr.getNetworkUsage((NetworkUsageCommand)cmd);
-            } else if (cmd instanceof IpAssocCommand) {
-                return _mockNetworkMgr.IpAssoc((IpAssocCommand)cmd);
-            } else if (cmd instanceof LoadBalancerConfigCommand) {
-                return _mockNetworkMgr.LoadBalancerConfig((LoadBalancerConfigCommand)cmd);
-            } else if (cmd instanceof DhcpEntryCommand) {
-                return _mockNetworkMgr.AddDhcpEntry((DhcpEntryCommand)cmd);
-            } else if (cmd instanceof VmDataCommand) {
-                return _mockVmMgr.setVmData((VmDataCommand)cmd);
-            } else if (cmd instanceof CleanupNetworkRulesCmd) {
-                return _mockVmMgr.CleanupNetworkRules((CleanupNetworkRulesCmd)cmd, info);
-            } else if (cmd instanceof CheckNetworkCommand) {
-                return _mockAgentMgr.checkNetworkCommand((CheckNetworkCommand)cmd);
-            } else if (cmd instanceof StopCommand) {
-                return _mockVmMgr.stopVM((StopCommand)cmd);
-            } else if (cmd instanceof RebootCommand) {
-                return _mockVmMgr.rebootVM((RebootCommand)cmd);
-            } else if (cmd instanceof GetVncPortCommand) {
-                return _mockVmMgr.getVncPort((GetVncPortCommand)cmd);
-            } else if (cmd instanceof CheckConsoleProxyLoadCommand) {
-                return _mockVmMgr.CheckConsoleProxyLoad((CheckConsoleProxyLoadCommand)cmd);
-            } else if (cmd instanceof WatchConsoleProxyLoadCommand) {
-                return _mockVmMgr.WatchConsoleProxyLoad((WatchConsoleProxyLoadCommand)cmd);
-            } else if (cmd instanceof SecurityGroupRulesCmd) {
-                return _mockVmMgr.AddSecurityGroupRules((SecurityGroupRulesCmd)cmd, info);
-            } else if (cmd instanceof SavePasswordCommand) {
-                return _mockVmMgr.SavePassword((SavePasswordCommand)cmd);
-            } else if (cmd instanceof PrimaryStorageDownloadCommand) {
-                return _mockStorageMgr.primaryStorageDownload((PrimaryStorageDownloadCommand)cmd);
-            } else if (cmd instanceof CreateCommand) {
-                return _mockStorageMgr.createVolume((CreateCommand)cmd);
-            } else if (cmd instanceof AttachVolumeCommand) {
-                return _mockStorageMgr.AttachVolume((AttachVolumeCommand)cmd);
-            } else if (cmd instanceof AttachIsoCommand) {
-                return _mockStorageMgr.AttachIso((AttachIsoCommand)cmd);
-            } else if (cmd instanceof DeleteStoragePoolCommand) {
-                return _mockStorageMgr.DeleteStoragePool((DeleteStoragePoolCommand)cmd);
-            } else if (cmd instanceof ModifyStoragePoolCommand) {
-                return _mockStorageMgr.ModifyStoragePool((ModifyStoragePoolCommand)cmd);
-            } else if (cmd instanceof CreateStoragePoolCommand) {
-                return _mockStorageMgr.CreateStoragePool((CreateStoragePoolCommand)cmd);
-            } else if (cmd instanceof SecStorageSetupCommand) {
-                return _mockStorageMgr.SecStorageSetup((SecStorageSetupCommand)cmd);
-            } else if (cmd instanceof ListTemplateCommand) {
-                return _mockStorageMgr.ListTemplates((ListTemplateCommand)cmd);
-            } else if (cmd instanceof ListVolumeCommand) {
-                return _mockStorageMgr.ListVolumes((ListVolumeCommand)cmd);
-            } else if (cmd instanceof DestroyCommand) {
-                return _mockStorageMgr.Destroy((DestroyCommand)cmd);
-            } else if (cmd instanceof DownloadProgressCommand) {
-                return _mockStorageMgr.DownloadProcess((DownloadProgressCommand)cmd);
-            } else if (cmd instanceof DownloadCommand) {
-                return _mockStorageMgr.Download((DownloadCommand)cmd);
-            } else if (cmd instanceof GetStorageStatsCommand) {
-                return _mockStorageMgr.GetStorageStats((GetStorageStatsCommand)cmd);
-            } else if (cmd instanceof ManageSnapshotCommand) {
-                return _mockStorageMgr.ManageSnapshot((ManageSnapshotCommand)cmd);
-            } else if (cmd instanceof BackupSnapshotCommand) {
-                return _mockStorageMgr.BackupSnapshot((BackupSnapshotCommand)cmd, info);
-            } else if (cmd instanceof CreateVolumeFromSnapshotCommand) {
-                return _mockStorageMgr.CreateVolumeFromSnapshot((CreateVolumeFromSnapshotCommand)cmd);
-            } else if (cmd instanceof DeleteCommand) {
-                return _mockStorageMgr.Delete((DeleteCommand)cmd);
-            } else if (cmd instanceof SecStorageVMSetupCommand) {
-                return _mockStorageMgr.SecStorageVMSetup((SecStorageVMSetupCommand)cmd);
-            } else if (cmd instanceof CreatePrivateTemplateFromSnapshotCommand) {
-                return _mockStorageMgr.CreatePrivateTemplateFromSnapshot((CreatePrivateTemplateFromSnapshotCommand)cmd);
-            } else if (cmd instanceof ComputeChecksumCommand) {
-                return _mockStorageMgr.ComputeChecksum((ComputeChecksumCommand)cmd);
-            } else if (cmd instanceof CreatePrivateTemplateFromVolumeCommand) {
-                return _mockStorageMgr.CreatePrivateTemplateFromVolume((CreatePrivateTemplateFromVolumeCommand)cmd);
-            } else if (cmd instanceof MaintainCommand) {
-                return _mockAgentMgr.maintain((MaintainCommand)cmd);
-            } else if (cmd instanceof GetVmStatsCommand) {
-                return _mockVmMgr.getVmStats((GetVmStatsCommand)cmd);
-            } else if (cmd instanceof CheckRouterCommand) {
-                return _mockVmMgr.checkRouter((CheckRouterCommand)cmd);
-            } else if (cmd instanceof BumpUpPriorityCommand) {
-                return _mockVmMgr.bumpPriority((BumpUpPriorityCommand)cmd);
-            } else if (cmd instanceof GetDomRVersionCmd) {
-                return _mockVmMgr.getDomRVersion((GetDomRVersionCmd)cmd);
-            } else if (cmd instanceof ClusterSyncCommand) {
-                return new Answer(cmd);
-            } else if (cmd instanceof CopyVolumeCommand) {
-                return _mockStorageMgr.CopyVolume((CopyVolumeCommand)cmd);
-            } else if (cmd instanceof PlugNicCommand) {
-                return _mockNetworkMgr.plugNic((PlugNicCommand)cmd);
-            } else if (cmd instanceof UnPlugNicCommand) {
-                return _mockNetworkMgr.unplugNic((UnPlugNicCommand)cmd);
-            } else if (cmd instanceof IpAssocVpcCommand) {
-                return _mockNetworkMgr.ipAssoc((IpAssocVpcCommand)cmd);
-            } else if (cmd instanceof SetSourceNatCommand) {
-                return _mockNetworkMgr.setSourceNat((SetSourceNatCommand)cmd);
-            } else if (cmd instanceof SetNetworkACLCommand) {
-                return _mockNetworkMgr.setNetworkAcl((SetNetworkACLCommand)cmd);
-            } else if (cmd instanceof SetupGuestNetworkCommand) {
-                return _mockNetworkMgr.setUpGuestNetwork((SetupGuestNetworkCommand)cmd);
-            } else if (cmd instanceof SetPortForwardingRulesVpcCommand) {
-                return _mockNetworkMgr.setVpcPortForwards((SetPortForwardingRulesVpcCommand)cmd);
-            } else if (cmd instanceof SetStaticNatRulesCommand) {
-                return _mockNetworkMgr.setVPCStaticNatRules((SetStaticNatRulesCommand)cmd);
-            } else if (cmd instanceof SetStaticRouteCommand) {
-                return _mockNetworkMgr.setStaticRoute((SetStaticRouteCommand)cmd);
-            } else if (cmd instanceof Site2SiteVpnCfgCommand) {
-                return _mockNetworkMgr.siteToSiteVpn((Site2SiteVpnCfgCommand)cmd);
-            } else if (cmd instanceof CheckS2SVpnConnectionsCommand) {
-                return _mockNetworkMgr.checkSiteToSiteVpnConnection((CheckS2SVpnConnectionsCommand)cmd);
-            } else if (cmd instanceof CreateVMSnapshotCommand) {
-                return _mockVmMgr.createVmSnapshot((CreateVMSnapshotCommand)cmd);
-            } else if (cmd instanceof DeleteVMSnapshotCommand) {
-                return _mockVmMgr.deleteVmSnapshot((DeleteVMSnapshotCommand)cmd);
-            } else if (cmd instanceof RevertToVMSnapshotCommand) {
-                return _mockVmMgr.revertVmSnapshot((RevertToVMSnapshotCommand)cmd);
-            } else if (cmd instanceof NetworkRulesVmSecondaryIpCommand) {
-                return _mockVmMgr.plugSecondaryIp((NetworkRulesVmSecondaryIpCommand)cmd);
-            } else if (cmd instanceof ScaleVmCommand) {
-                return _mockVmMgr.scaleVm((ScaleVmCommand)cmd);
-            } else if (cmd instanceof PvlanSetupCommand) {
-                return _mockNetworkMgr.setupPVLAN((PvlanSetupCommand)cmd);
-            } else if (cmd instanceof StorageSubSystemCommand) {
-                return this.storageHandler.handleStorageCommands((StorageSubSystemCommand)cmd);
-            } else if (cmd instanceof GetRouterAlertsCommand) {
-                return new Answer(cmd);
-            } else if (cmd instanceof VpnUsersCfgCommand || cmd instanceof RemoteAccessVpnCfgCommand || cmd instanceof SetMonitorServiceCommand || cmd instanceof AggregationControlCommand) {
-                return new Answer(cmd);
-            } else {
-                s_logger.error("Simulator does not implement command of type " + cmd.toString());
-                return Answer.createUnsupportedCommandAnswer(cmd);
+            if (answer == null) {
+                if (cmd instanceof GetHostStatsCommand) {
+                    answer = _mockAgentMgr.getHostStatistic((GetHostStatsCommand)cmd);
+                } else if (cmd instanceof CheckHealthCommand) {
+                    answer = _mockAgentMgr.checkHealth((CheckHealthCommand)cmd);
+                } else if (cmd instanceof PingTestCommand) {
+                    answer = _mockAgentMgr.pingTest((PingTestCommand)cmd);
+                } else if (cmd instanceof PrepareForMigrationCommand) {
+                    answer = _mockVmMgr.prepareForMigrate((PrepareForMigrationCommand)cmd);
+                } else if (cmd instanceof MigrateCommand) {
+                    answer = _mockVmMgr.Migrate((MigrateCommand)cmd, info);
+                } else if (cmd instanceof StartCommand) {
+                    answer = _mockVmMgr.startVM((StartCommand)cmd, info);
+                } else if (cmd instanceof CheckSshCommand) {
+                    answer = _mockVmMgr.checkSshCommand((CheckSshCommand)cmd);
+                } else if (cmd instanceof CheckVirtualMachineCommand) {
+                    answer = _mockVmMgr.checkVmState((CheckVirtualMachineCommand)cmd);
+                } else if (cmd instanceof SetStaticNatRulesCommand) {
+                    answer = _mockNetworkMgr.SetStaticNatRules((SetStaticNatRulesCommand)cmd);
+                } else if (cmd instanceof SetFirewallRulesCommand) {
+                    answer = _mockNetworkMgr.SetFirewallRules((SetFirewallRulesCommand)cmd);
+                } else if (cmd instanceof SetPortForwardingRulesCommand) {
+                    answer = _mockNetworkMgr.SetPortForwardingRules((SetPortForwardingRulesCommand)cmd);
+                } else if (cmd instanceof NetworkUsageCommand) {
+                    answer = _mockNetworkMgr.getNetworkUsage((NetworkUsageCommand)cmd);
+                } else if (cmd instanceof IpAssocCommand) {
+                    answer = _mockNetworkMgr.IpAssoc((IpAssocCommand)cmd);
+                } else if (cmd instanceof LoadBalancerConfigCommand) {
+                    answer = _mockNetworkMgr.LoadBalancerConfig((LoadBalancerConfigCommand)cmd);
+                } else if (cmd instanceof DhcpEntryCommand) {
+                    answer = _mockNetworkMgr.AddDhcpEntry((DhcpEntryCommand)cmd);
+                } else if (cmd instanceof VmDataCommand) {
+                    answer = _mockVmMgr.setVmData((VmDataCommand)cmd);
+                } else if (cmd instanceof CleanupNetworkRulesCmd) {
+                    answer = _mockVmMgr.CleanupNetworkRules((CleanupNetworkRulesCmd)cmd, info);
+                } else if (cmd instanceof CheckNetworkCommand) {
+                    answer = _mockAgentMgr.checkNetworkCommand((CheckNetworkCommand)cmd);
+                } else if (cmd instanceof StopCommand) {
+                    answer = _mockVmMgr.stopVM((StopCommand)cmd);
+                } else if (cmd instanceof RebootCommand) {
+                    answer = _mockVmMgr.rebootVM((RebootCommand)cmd);
+                } else if (cmd instanceof GetVncPortCommand) {
+                    answer = _mockVmMgr.getVncPort((GetVncPortCommand)cmd);
+                } else if (cmd instanceof CheckConsoleProxyLoadCommand) {
+                    answer = _mockVmMgr.CheckConsoleProxyLoad((CheckConsoleProxyLoadCommand)cmd);
+                } else if (cmd instanceof WatchConsoleProxyLoadCommand) {
+                    answer = _mockVmMgr.WatchConsoleProxyLoad((WatchConsoleProxyLoadCommand)cmd);
+                } else if (cmd instanceof SecurityGroupRulesCmd) {
+                    answer = _mockVmMgr.AddSecurityGroupRules((SecurityGroupRulesCmd)cmd, info);
+                } else if (cmd instanceof SavePasswordCommand) {
+                    answer = _mockVmMgr.SavePassword((SavePasswordCommand)cmd);
+                } else if (cmd instanceof PrimaryStorageDownloadCommand) {
+                    answer = _mockStorageMgr.primaryStorageDownload((PrimaryStorageDownloadCommand)cmd);
+                } else if (cmd instanceof CreateCommand) {
+                    answer = _mockStorageMgr.createVolume((CreateCommand)cmd);
+                } else if (cmd instanceof AttachVolumeCommand) {
+                    answer = _mockStorageMgr.AttachVolume((AttachVolumeCommand)cmd);
+                } else if (cmd instanceof AttachIsoCommand) {
+                    answer = _mockStorageMgr.AttachIso((AttachIsoCommand)cmd);
+                } else if (cmd instanceof DeleteStoragePoolCommand) {
+                    answer = _mockStorageMgr.DeleteStoragePool((DeleteStoragePoolCommand)cmd);
+                } else if (cmd instanceof ModifyStoragePoolCommand) {
+                    answer = _mockStorageMgr.ModifyStoragePool((ModifyStoragePoolCommand)cmd);
+                } else if (cmd instanceof CreateStoragePoolCommand) {
+                    answer = _mockStorageMgr.CreateStoragePool((CreateStoragePoolCommand)cmd);
+                } else if (cmd instanceof SecStorageSetupCommand) {
+                    answer = _mockStorageMgr.SecStorageSetup((SecStorageSetupCommand)cmd);
+                } else if (cmd instanceof ListTemplateCommand) {
+                    answer = _mockStorageMgr.ListTemplates((ListTemplateCommand)cmd);
+                } else if (cmd instanceof ListVolumeCommand) {
+                    answer = _mockStorageMgr.ListVolumes((ListVolumeCommand)cmd);
+                } else if (cmd instanceof DestroyCommand) {
+                    answer = _mockStorageMgr.Destroy((DestroyCommand)cmd);
+                } else if (cmd instanceof DownloadProgressCommand) {
+                    answer = _mockStorageMgr.DownloadProcess((DownloadProgressCommand)cmd);
+                } else if (cmd instanceof DownloadCommand) {
+                    answer = _mockStorageMgr.Download((DownloadCommand)cmd);
+                } else if (cmd instanceof GetStorageStatsCommand) {
+                    answer = _mockStorageMgr.GetStorageStats((GetStorageStatsCommand)cmd);
+                } else if (cmd instanceof ManageSnapshotCommand) {
+                    answer = _mockStorageMgr.ManageSnapshot((ManageSnapshotCommand)cmd);
+                } else if (cmd instanceof BackupSnapshotCommand) {
+                    answer = _mockStorageMgr.BackupSnapshot((BackupSnapshotCommand)cmd, info);
+                } else if (cmd instanceof CreateVolumeFromSnapshotCommand) {
+                    answer = _mockStorageMgr.CreateVolumeFromSnapshot((CreateVolumeFromSnapshotCommand)cmd);
+                } else if (cmd instanceof DeleteCommand) {
+                    answer = _mockStorageMgr.Delete((DeleteCommand)cmd);
+                } else if (cmd instanceof SecStorageVMSetupCommand) {
+                    answer = _mockStorageMgr.SecStorageVMSetup((SecStorageVMSetupCommand)cmd);
+                } else if (cmd instanceof CreatePrivateTemplateFromSnapshotCommand) {
+                    answer = _mockStorageMgr.CreatePrivateTemplateFromSnapshot((CreatePrivateTemplateFromSnapshotCommand)cmd);
+                } else if (cmd instanceof ComputeChecksumCommand) {
+                    answer = _mockStorageMgr.ComputeChecksum((ComputeChecksumCommand)cmd);
+                } else if (cmd instanceof CreatePrivateTemplateFromVolumeCommand) {
+                    answer = _mockStorageMgr.CreatePrivateTemplateFromVolume((CreatePrivateTemplateFromVolumeCommand)cmd);
+                } else if (cmd instanceof MaintainCommand) {
+                    answer = _mockAgentMgr.maintain((MaintainCommand)cmd);
+                } else if (cmd instanceof GetVmStatsCommand) {
+                    answer = _mockVmMgr.getVmStats((GetVmStatsCommand)cmd);
+                } else if (cmd instanceof CheckRouterCommand) {
+                    answer = _mockVmMgr.checkRouter((CheckRouterCommand)cmd);
+                } else if (cmd instanceof BumpUpPriorityCommand) {
+                    answer = _mockVmMgr.bumpPriority((BumpUpPriorityCommand)cmd);
+                } else if (cmd instanceof GetDomRVersionCmd) {
+                    answer = _mockVmMgr.getDomRVersion((GetDomRVersionCmd)cmd);
+                } else if (cmd instanceof ClusterSyncCommand) {
+                    answer = new Answer(cmd);
+                } else if (cmd instanceof CopyVolumeCommand) {
+                    answer = _mockStorageMgr.CopyVolume((CopyVolumeCommand)cmd);
+                } else if (cmd instanceof PlugNicCommand) {
+                    answer = _mockNetworkMgr.plugNic((PlugNicCommand)cmd);
+                } else if (cmd instanceof UnPlugNicCommand) {
+                    answer = _mockNetworkMgr.unplugNic((UnPlugNicCommand)cmd);
+                } else if (cmd instanceof IpAssocVpcCommand) {
+                    answer = _mockNetworkMgr.ipAssoc((IpAssocVpcCommand)cmd);
+                } else if (cmd instanceof SetSourceNatCommand) {
+                    answer = _mockNetworkMgr.setSourceNat((SetSourceNatCommand)cmd);
+                } else if (cmd instanceof SetNetworkACLCommand) {
+                    answer = _mockNetworkMgr.setNetworkAcl((SetNetworkACLCommand)cmd);
+                } else if (cmd instanceof SetupGuestNetworkCommand) {
+                    answer = _mockNetworkMgr.setUpGuestNetwork((SetupGuestNetworkCommand)cmd);
+                } else if (cmd instanceof SetPortForwardingRulesVpcCommand) {
+                    answer = _mockNetworkMgr.setVpcPortForwards((SetPortForwardingRulesVpcCommand)cmd);
+                } else if (cmd instanceof SetStaticNatRulesCommand) {
+                    answer = _mockNetworkMgr.setVPCStaticNatRules((SetStaticNatRulesCommand)cmd);
+                } else if (cmd instanceof SetStaticRouteCommand) {
+                    answer = _mockNetworkMgr.setStaticRoute((SetStaticRouteCommand)cmd);
+                } else if (cmd instanceof Site2SiteVpnCfgCommand) {
+                    answer = _mockNetworkMgr.siteToSiteVpn((Site2SiteVpnCfgCommand)cmd);
+                } else if (cmd instanceof CheckS2SVpnConnectionsCommand) {
+                    answer = _mockNetworkMgr.checkSiteToSiteVpnConnection((CheckS2SVpnConnectionsCommand)cmd);
+                } else if (cmd instanceof CreateVMSnapshotCommand) {
+                    answer = _mockVmMgr.createVmSnapshot((CreateVMSnapshotCommand)cmd);
+                } else if (cmd instanceof DeleteVMSnapshotCommand) {
+                    answer = _mockVmMgr.deleteVmSnapshot((DeleteVMSnapshotCommand)cmd);
+                } else if (cmd instanceof RevertToVMSnapshotCommand) {
+                    answer = _mockVmMgr.revertVmSnapshot((RevertToVMSnapshotCommand)cmd);
+                } else if (cmd instanceof NetworkRulesVmSecondaryIpCommand) {
+                    answer = _mockVmMgr.plugSecondaryIp((NetworkRulesVmSecondaryIpCommand)cmd);
+                } else if (cmd instanceof ScaleVmCommand) {
+                    answer = _mockVmMgr.scaleVm((ScaleVmCommand)cmd);
+                } else if (cmd instanceof PvlanSetupCommand) {
+                    answer = _mockNetworkMgr.setupPVLAN((PvlanSetupCommand)cmd);
+                } else if (cmd instanceof StorageSubSystemCommand) {
+                    answer = this.storageHandler.handleStorageCommands((StorageSubSystemCommand)cmd);
+                } else if (cmd instanceof GetRouterAlertsCommand) {
+                    answer = new Answer(cmd);
+                } else if (cmd instanceof VpnUsersCfgCommand || cmd instanceof RemoteAccessVpnCfgCommand || cmd instanceof SetMonitorServiceCommand || cmd instanceof AggregationControlCommand) {
+                    answer = new Answer(cmd);
+                } else {
+                    s_logger.error("Simulator does not implement command of type " + cmd.toString());
+                    answer = Answer.createUnsupportedCommandAnswer(cmd);
+                }
             }
+
+            if (config != null && (config.getCount() != null && config.getCount().intValue() > 0)) {
+                if (answer != null) {
+                    config.setCount(config.getCount().intValue() - 1);
+                    _mockConfigDao.update(config.getId(), config);
+                }
+            }
+
+            return answer;
         } catch (Exception e) {
             s_logger.error("Failed execute cmd: ", e);
             txn.rollback();
@@ -413,7 +465,8 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage
     }
 
     @Override
-    public boolean configureSimulator(Long zoneId, Long podId, Long clusterId, Long hostId, String command, String values) {
+    public Long configureSimulator(Long zoneId, Long podId, Long clusterId, Long hostId, String command, String values, Integer count, String jsonResponse) {
+        Long id = null;
         TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
         try {
             txn.start();
@@ -426,21 +479,71 @@ public class SimulatorManagerImpl extends ManagerBase implements SimulatorManage
                 config.setHostId(hostId);
                 config.setName(command);
                 config.setValues(values);
-                _mockConfigDao.persist(config);
+                config.setCount(count);
+                config.setJsonResponse(jsonResponse);
+                config = _mockConfigDao.persist(config);
                 txn.commit();
             } else {
                 config.setValues(values);
+                config.setCount(count);
+                config.setJsonResponse(jsonResponse);
                 _mockConfigDao.update(config.getId(), config);
                 txn.commit();
             }
+            id = config.getId();
         } catch (Exception ex) {
             txn.rollback();
-            throw new CloudRuntimeException("Unable to configure simulator because of " + ex.getMessage(), ex);
+            throw new CloudRuntimeException("Unable to configure simulator mock because of " + ex.getMessage(), ex);
         } finally {
             txn.close();
             txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
             txn.close();
         }
-        return true;
+        return id;
+    }
+
+    @Override
+    public MockConfigurationVO querySimulatorMock(Long id) {
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
+        try {
+            txn.start();
+            return _mockConfigDao.findById(id);
+        } catch (Exception ex) {
+            txn.rollback();
+            throw new CloudRuntimeException("Unable to query simulator mock because of " + ex.getMessage(), ex);
+        } finally {
+            txn.close();
+            txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
+            txn.close();
+        }
+    }
+
+    @Override
+    public boolean clearSimulatorMock(Long id) {
+        boolean status = false;
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
+        try {
+            txn.start();
+            MockConfigurationVO config = _mockConfigDao.findById(id);
+            if (config != null) {
+                config.setRemoved(new Date());
+                _mockConfigDao.update(config.getId(), config);
+                status = true;
+                txn.commit();
+            }
+        } catch (Exception ex) {
+            txn.rollback();
+            throw new CloudRuntimeException("Unable to cleanup simulator mock because of " + ex.getMessage(), ex);
+        } finally {
+            txn.close();
+            txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
+            txn.close();
+        }
+        return status;
+    }
+
+    @Override
+    public MockConfigurationDao getMockConfigurationDao() {
+        return _mockConfigDao;
     }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/api/commands/CleanupSimulatorMockCmd.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/api/commands/CleanupSimulatorMockCmd.java b/plugins/hypervisors/simulator/src/com/cloud/api/commands/CleanupSimulatorMockCmd.java
new file mode 100644
index 0000000..3aabb41
--- /dev/null
+++ b/plugins/hypervisors/simulator/src/com/cloud/api/commands/CleanupSimulatorMockCmd.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.api.commands;
+
+import com.cloud.agent.manager.SimulatorManager;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.user.Account;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.log4j.Logger;
+
+import javax.inject.Inject;
+
+
+@APICommand(name = "cleanupSimulatorMock", description="cleanup simulator mock", responseObject=SuccessResponse.class)
+public class CleanupSimulatorMockCmd extends BaseCmd {
+    public static final Logger s_logger = Logger.getLogger(CleanupSimulatorMockCmd.class.getName());
+    private static final String s_name = "cleanupsimulatormockresponse";
+
+    @Inject SimulatorManager _simMgr;
+
+    @Parameter(name=ApiConstants.ID, type=CommandType.LONG, description="id of the configured mock")
+    private Long id;
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException {
+        boolean result = _simMgr.clearSimulatorMock(id);
+        if (!result) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to cleanup mock");
+        }
+
+        SuccessResponse response = new SuccessResponse(getCommandName());
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return s_name;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java b/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java
old mode 100755
new mode 100644
index 65bc7be..9498a93
--- a/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/api/commands/ConfigureSimulatorCmd.java
@@ -19,23 +19,27 @@ package com.cloud.api.commands;
 import javax.inject.Inject;
 
 import org.apache.log4j.Logger;
-
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiErrorCode;
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.PodResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
 
 import com.cloud.agent.manager.SimulatorManager;
+import com.cloud.api.response.MockResponse;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.simulator.MockConfigurationVO;
 import com.cloud.user.Account;
 
-@APICommand(name = "configureSimulator", description = "configure simulator", responseObject = SuccessResponse.class,
+@APICommand(name = "configureSimulator", description = "configure simulator", responseObject = MockResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
 public class ConfigureSimulatorCmd extends BaseCmd {
     public static final Logger s_logger = Logger.getLogger(ConfigureSimulatorCmd.class.getName());
@@ -44,16 +48,16 @@ public class ConfigureSimulatorCmd extends BaseCmd {
     @Inject
     SimulatorManager _simMgr;
 
-    @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.LONG, description = "configure range: in a zone")
+    @Parameter(name=ApiConstants.ZONE_ID, type=CommandType.UUID, entityType=ZoneResponse.class, description="configure range: in a zone")
     private Long zoneId;
 
-    @Parameter(name = ApiConstants.POD_ID, type = CommandType.LONG, description = "configure range: in a pod")
+    @Parameter(name=ApiConstants.POD_ID, type=CommandType.UUID, entityType=PodResponse.class, description="configure range: in a pod")
     private Long podId;
 
-    @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.LONG, description = "configure range: in a cluster")
+    @Parameter(name=ApiConstants.CLUSTER_ID, type=CommandType.UUID, entityType=ClusterResponse.class, description="configure range: in a cluster")
     private Long clusterId;
 
-    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.LONG, description = "configure range: in a host")
+    @Parameter(name=ApiConstants.HOST_ID, type=CommandType.UUID, entityType=HostResponse.class, description="configure range: in a host")
     private Long hostId;
 
     @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "which command needs to be configured")
@@ -62,15 +66,34 @@ public class ConfigureSimulatorCmd extends BaseCmd {
     @Parameter(name = ApiConstants.VALUE, type = CommandType.STRING, required = true, description = "configuration options for this command, which is seperated by ;")
     private String values;
 
+    @Parameter(name=ApiConstants.COUNT, type=CommandType.INTEGER, description="number of times the mock is active")
+    private Integer count;
+
+    @Parameter(name="jsonresponse", type=CommandType.STRING, description="agent command response to be returned")
+    private String jsonResponse;
+
     @Override
     public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
         ResourceAllocationException {
-        boolean result = _simMgr.configureSimulator(zoneId, podId, clusterId, hostId, command, values);
-        if (!result) {
+        Long id = _simMgr.configureSimulator(zoneId, podId, clusterId, hostId, command, values, count, jsonResponse);
+        if (id == null) {
             throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to configure simulator");
         }
 
-        SuccessResponse response = new SuccessResponse(getCommandName());
+        MockConfigurationVO config = _simMgr.querySimulatorMock(id);
+        if (config == null) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to query simulator mock");
+        }
+
+        MockResponse response = new MockResponse();
+        response.setId(config.getId());
+        response.setZoneId(config.getDataCenterId());
+        response.setPodId(config.getPodId());
+        response.setClusterId(config.getClusterId());
+        response.setHostId(config.getHostId());
+        response.setName(config.getName());
+        response.setCount(config.getCount());
+        response.setResponseName("simulatormock");
         this.setResponseObject(response);
     }
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/api/commands/QuerySimulatorMockCmd.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/api/commands/QuerySimulatorMockCmd.java b/plugins/hypervisors/simulator/src/com/cloud/api/commands/QuerySimulatorMockCmd.java
new file mode 100644
index 0000000..15ee7f7
--- /dev/null
+++ b/plugins/hypervisors/simulator/src/com/cloud/api/commands/QuerySimulatorMockCmd.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.api.commands;
+
+import com.cloud.agent.manager.SimulatorManager;
+import com.cloud.api.response.MockResponse;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.simulator.MockConfigurationVO;
+import com.cloud.user.Account;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.log4j.Logger;
+
+import javax.inject.Inject;
+
+
+@APICommand(name = "querySimulatorMock", description="query simulator mock", responseObject=MockResponse.class)
+public class QuerySimulatorMockCmd extends BaseCmd {
+    public static final Logger s_logger = Logger.getLogger(QuerySimulatorMockCmd.class.getName());
+    private static final String s_name = "querysimulatormockresponse";
+
+    @Inject SimulatorManager _simMgr;
+
+    @Parameter(name=ApiConstants.ID, type=CommandType.LONG, description="id of the configured mock")
+    private Long id;
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException {
+        MockConfigurationVO config = _simMgr.querySimulatorMock(id);
+        if (config == null) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to query mock");
+        }
+
+        MockResponse response = new MockResponse();
+        response.setId(config.getId());
+        response.setZoneId(config.getDataCenterId());
+        response.setPodId(config.getPodId());
+        response.setClusterId(config.getClusterId());
+        response.setHostId(config.getHostId());
+        response.setName(config.getName());
+        response.setCount(config.getCount());
+        response.setResponseName("simulatormock");
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public String getCommandName() {
+        return s_name;
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/api/response/MockResponse.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/api/response/MockResponse.java b/plugins/hypervisors/simulator/src/com/cloud/api/response/MockResponse.java
new file mode 100644
index 0000000..130d5de
--- /dev/null
+++ b/plugins/hypervisors/simulator/src/com/cloud/api/response/MockResponse.java
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.api.response;
+
+import com.cloud.serializer.Param;
+import com.cloud.simulator.MockConfigurationVO;
+import com.google.gson.annotations.SerializedName;
+
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.api.EntityReference;
+
+
+@EntityReference(value = MockConfigurationVO.class)
+public class MockResponse extends BaseResponse {
+    @SerializedName(ApiConstants.ID) @Param(description="the mock ID")
+    private Long id;
+
+    @SerializedName(ApiConstants.ZONE_ID) @Param(description="the Zone ID scope of the mock")
+    private Long zoneId;
+
+    @SerializedName(ApiConstants.POD_ID) @Param(description="the Pod ID scope of the mock")
+    private Long podId;
+
+    @SerializedName(ApiConstants.CLUSTER_ID) @Param(description="the Cluster ID scope of the mock")
+    private Long clusterId;
+
+    @SerializedName(ApiConstants.HOST_ID) @Param(description="the Host ID scope of the mock")
+    private Long hostId;
+
+    @SerializedName(ApiConstants.NAME) @Param(description="the agent command to be mocked")
+    private String name;
+
+    @SerializedName(ApiConstants.COUNT) @Param(description="number of times mock is executed, if not specified then mock remains active till cleaned up")
+    private Integer count;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public Long getZoneId() {
+        return zoneId;
+    }
+
+    public void setZoneId(Long zoneId) {
+        this.zoneId = zoneId;
+    }
+
+    public Long getPodId() {
+        return podId;
+    }
+
+    public void setPodId(Long podId) {
+        this.podId = podId;
+    }
+
+    public Long getClusterId() {
+        return clusterId;
+    }
+
+    public void setClusterId(Long clusterId) {
+        this.clusterId = clusterId;
+    }
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public void setHostId(Long hostId) {
+        this.hostId = hostId;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public Integer getCount() {
+        return count;
+    }
+
+    public void setCount(Integer count) {
+        this.count = count;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorFencer.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorFencer.java b/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorFencer.java
new file mode 100644
index 0000000..aa7c3d4
--- /dev/null
+++ b/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorFencer.java
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.ha;
+
+import java.util.List;
+import java.util.Map;
+
+import javax.ejb.Local;
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.FenceAnswer;
+import com.cloud.agent.api.FenceCommand;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.resource.ResourceManager;
+import com.cloud.utils.component.AdapterBase;
+import com.cloud.vm.VirtualMachine;
+
+@Local(value=FenceBuilder.class)
+public class SimulatorFencer extends AdapterBase implements FenceBuilder {
+    private static final Logger s_logger = Logger.getLogger(SimulatorFencer.class);
+
+    @Inject HostDao _hostDao;
+    @Inject AgentManager _agentMgr;
+    @Inject ResourceManager _resourceMgr;
+    @Override
+    public boolean configure(String name, Map<String, Object> params)
+            throws ConfigurationException {
+        // TODO Auto-generated method stub
+        return true;
+    }
+
+    @Override
+    public boolean start() {
+        // TODO Auto-generated method stub
+        return true;
+    }
+
+    @Override
+    public boolean stop() {
+        // TODO Auto-generated method stub
+        return true;
+    }
+
+    public SimulatorFencer() {
+        super();
+    }
+
+    @Override
+    public Boolean fenceOff(VirtualMachine vm, Host host) {
+        if (host.getHypervisorType() != HypervisorType.Simulator) {
+            s_logger.debug("Don't know how to fence non simulator hosts " + host.getHypervisorType());
+            return null;
+        }
+
+        List<HostVO> hosts = _resourceMgr.listAllHostsInCluster(host.getClusterId());
+        FenceCommand fence = new FenceCommand(vm, host);
+
+        for (HostVO h : hosts) {
+            if (h.getHypervisorType() == HypervisorType.Simulator) {
+                if( h.getStatus() != Status.Up ) {
+                    continue;
+                }
+                if( h.getId() == host.getId() ) {
+                    continue;
+                }
+                FenceAnswer answer = null;
+                try {
+                    answer = (FenceAnswer)_agentMgr.send(h.getId(), fence);
+                } catch (AgentUnavailableException e) {
+                    if (s_logger.isDebugEnabled()) {
+                        s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
+                    }
+                    continue;
+                } catch (OperationTimedoutException e) {
+                    if (s_logger.isDebugEnabled()) {
+                        s_logger.debug("Moving on to the next host because " + h.toString() + " is unavailable");
+                    }
+                    continue;
+                }
+                if (answer != null && answer.getResult()) {
+                    return true;
+                }
+            }
+        }
+
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("Unable to fence off " + vm.toString() + " on " + host.toString());
+        }
+
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorInvestigator.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorInvestigator.java b/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorInvestigator.java
new file mode 100644
index 0000000..448d7ed
--- /dev/null
+++ b/plugins/hypervisors/simulator/src/com/cloud/ha/SimulatorInvestigator.java
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.ha;
+
+import java.util.List;
+
+import javax.ejb.Local;
+import javax.inject.Inject;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.CheckOnHostCommand;
+import com.cloud.agent.api.CheckVirtualMachineAnswer;
+import com.cloud.agent.api.CheckVirtualMachineCommand;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.resource.ResourceManager;
+import com.cloud.simulator.dao.MockConfigurationDao;
+import com.cloud.utils.component.AdapterBase;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachine.State;
+
+@Local(value=Investigator.class)
+public class SimulatorInvestigator extends AdapterBase implements Investigator {
+    private final static Logger s_logger = Logger.getLogger(SimulatorInvestigator.class);
+    @Inject
+    AgentManager _agentMgr;
+    @Inject
+    ResourceManager _resourceMgr;
+    @Inject
+    MockConfigurationDao _mockConfigDao;
+
+    protected SimulatorInvestigator() {
+    }
+
+    @Override
+    public Status isAgentAlive(Host agent) {
+        if (agent.getHypervisorType() != HypervisorType.Simulator) {
+            return null;
+        }
+
+        CheckOnHostCommand cmd = new CheckOnHostCommand(agent);
+        List<HostVO> neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up);
+        for (HostVO neighbor : neighbors) {
+            if (neighbor.getId() == agent.getId() || neighbor.getHypervisorType() != Hypervisor.HypervisorType.Simulator) {
+                continue;
+            }
+            try {
+                Answer answer = _agentMgr.easySend(neighbor.getId(), cmd);
+                if (answer != null) {
+                    return answer.getResult() ? Status.Up : Status.Down;
+                }
+            } catch (Exception e) {
+                s_logger.debug("Failed to send command to host: " + neighbor.getId());
+            }
+        }
+
+        return null;
+    }
+
+    @Override
+    public Boolean isVmAlive(VirtualMachine vm, Host host) {
+        CheckVirtualMachineCommand cmd = new CheckVirtualMachineCommand(vm.getInstanceName());
+        try {
+            Answer answer = _agentMgr.send(vm.getHostId(), cmd);
+            if (!answer.getResult()) {
+                s_logger.debug("Unable to get vm state on " + vm.toString());
+                return null;
+            }
+            CheckVirtualMachineAnswer cvmAnswer = (CheckVirtualMachineAnswer)answer;
+            s_logger.debug("Agent responded with state " + cvmAnswer.getState().toString());
+            return cvmAnswer.getState() == State.Running;
+        } catch (AgentUnavailableException e) {
+            s_logger.debug("Unable to reach the agent for " + vm.toString() + ": " + e.getMessage());
+            return null;
+        } catch (OperationTimedoutException e) {
+            s_logger.debug("Operation timed out for " + vm.toString() + ": " + e.getMessage());
+            return null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/resource/AgentRoutingResource.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/resource/AgentRoutingResource.java b/plugins/hypervisors/simulator/src/com/cloud/resource/AgentRoutingResource.java
index f857261..4b5c9fd 100644
--- a/plugins/hypervisors/simulator/src/com/cloud/resource/AgentRoutingResource.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/resource/AgentRoutingResource.java
@@ -50,10 +50,12 @@ import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.network.Networks.RouterPrivateIpStrategy;
+import com.cloud.simulator.MockConfigurationVO;
 import com.cloud.simulator.MockVMVO;
 import com.cloud.storage.Storage.StorageResourceType;
 import com.cloud.storage.template.TemplateProp;
 import com.cloud.utils.Pair;
+import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.vm.VirtualMachine.PowerState;
 import com.cloud.vm.VirtualMachine.State;
 
@@ -104,6 +106,28 @@ public class AgentRoutingResource extends AgentStorageResource {
 
     @Override
     public PingCommand getCurrentStatus(long id) {
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.SIMULATOR_DB);
+        try {
+            MockConfigurationVO config = _simMgr.getMockConfigurationDao().findByNameBottomUP(agentHost.getDataCenterId(), agentHost.getPodId(), agentHost.getClusterId(), agentHost.getId(), "PingCommand");
+            if (config != null) {
+                Map<String, String> configParameters = config.getParameters();
+                for (Map.Entry<String, String> entry : configParameters.entrySet()) {
+                    if (entry.getKey().equalsIgnoreCase("result")) {
+                        String value = entry.getValue();
+                        if (value.equalsIgnoreCase("fail")) {
+                            return null;
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            txn.rollback();
+        } finally {
+            txn.close();
+            txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
+            txn.close();
+        }
+
         if (isStopped()) {
             return null;
         }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorDiscoverer.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorDiscoverer.java b/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorDiscoverer.java
old mode 100755
new mode 100644

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/simulator/MockConfigurationVO.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/simulator/MockConfigurationVO.java b/plugins/hypervisors/simulator/src/com/cloud/simulator/MockConfigurationVO.java
index 585598a..1beff76 100644
--- a/plugins/hypervisors/simulator/src/com/cloud/simulator/MockConfigurationVO.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/simulator/MockConfigurationVO.java
@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.simulator;
 
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -25,6 +26,8 @@ import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
 import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
 
 import org.apache.cloudstack.api.InternalIdentity;
 
@@ -54,6 +57,16 @@ public class MockConfigurationVO implements InternalIdentity {
     @Column(name = "values")
     private String values;
 
+    @Column(name="count")
+    private Integer count;
+
+    @Column(name="json_response")
+    private String jsonResponse;
+
+    @Column(name="removed")
+    @Temporal(value=TemporalType.TIMESTAMP)
+    private Date removed;
+
     @Override
     public long getId() {
         return this.id;
@@ -120,4 +133,24 @@ public class MockConfigurationVO implements InternalIdentity {
     public void setValues(String values) {
         this.values = values;
     }
+
+    public Integer getCount() {
+        return this.count;
+    }
+
+    public void setCount(Integer count) {
+        this.count = count;
+    }
+
+    public String getJsonResponse() {
+        return this.jsonResponse;
+    }
+
+    public void setJsonResponse(String jsonResponse) {
+        this.jsonResponse = jsonResponse;
+    }
+
+    public void setRemoved(Date removed) {
+        this.removed = removed;
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/plugins/hypervisors/simulator/src/com/cloud/simulator/dao/MockConfigurationDaoImpl.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/com/cloud/simulator/dao/MockConfigurationDaoImpl.java b/plugins/hypervisors/simulator/src/com/cloud/simulator/dao/MockConfigurationDaoImpl.java
index 040025a..3141798 100644
--- a/plugins/hypervisors/simulator/src/com/cloud/simulator/dao/MockConfigurationDaoImpl.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/simulator/dao/MockConfigurationDaoImpl.java
@@ -127,7 +127,9 @@ public class MockConfigurationDaoImpl extends GenericDaoBase<MockConfigurationVO
         formatter.format(" or (data_center_id = %d and pod_id = %d and cluster_id = %d and host_id is null)", dcId, podId, clusterId);
         formatter.format(" or (data_center_id = %d and pod_id = %d and cluster_id is null and host_id is null)", dcId, podId);
         formatter.format(" or (data_center_id = %d and pod_id is null and cluster_id is null and host_id is null)", dcId);
-        formatter.format(" or (data_center_id is null and pod_id is null and cluster_id is null and host_id is null)) LIMIT 1");
+        formatter.format(" or (data_center_id is null and pod_id is null and cluster_id is null and host_id is null))");
+        formatter.format(" and removed is NULL ORDER BY id ASC LIMIT 1 for update");
+        formatter.close();
 
         PreparedStatement pstmt = null;
         try {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/setup/db/create-schema-simulator.sql
----------------------------------------------------------------------
diff --git a/setup/db/create-schema-simulator.sql b/setup/db/create-schema-simulator.sql
index facfefc..d33596e 100644
--- a/setup/db/create-schema-simulator.sql
+++ b/setup/db/create-schema-simulator.sql
@@ -106,7 +106,10 @@ CREATE TABLE `simulator`.`mockconfiguration` (
   `cluster_id` bigint unsigned,
   `host_id` bigint unsigned,
   `name` varchar(255),
-  `values` varchar(4095),
+  `values` varchar(4096),
+  `count` int,
+  `json_response` varchar(4096),
+  `removed` datetime,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/25b41597/test/integration/smoke/test_deploy_vm.py
----------------------------------------------------------------------
diff --git a/test/integration/smoke/test_deploy_vm.py b/test/integration/smoke/test_deploy_vm.py
index 9f3633c..f31bc11 100644
--- a/test/integration/smoke/test_deploy_vm.py
+++ b/test/integration/smoke/test_deploy_vm.py
@@ -23,7 +23,7 @@ from marvin.cloudstackTestCase import cloudstackTestCase
 #Import Integration Libraries
 
 #base - contains all resources as entities and defines create, delete, list operations on them
-from marvin.integration.lib.base import Account, VirtualMachine, ServiceOffering
+from marvin.integration.lib.base import Account, VirtualMachine, ServiceOffering, SimulatorMock
 
 #utils - utility classes for common cleanup, external library wrappers etc
 from marvin.integration.lib.utils import cleanup_resources
@@ -56,6 +56,10 @@ class TestData(object):
                 "name" : "testvm2",
                 "displayname" : "Test VM2",
             },
+            "virtual_machine3" : {
+                "name" : "testvm3",
+                "displayname" : "Test VM3",
+            },
             #small service offering
             "service_offering": {
                 "small": {
@@ -120,22 +124,11 @@ class TestDeployVM(cloudstackTestCase):
         )
 
         list_vms = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)
-
         self.debug(
             "Verify listVirtualMachines response for virtual machine: %s"\
             % self.virtual_machine.id
         )
-
-        self.assertEqual(
-            isinstance(list_vms, list),
-            True,
-            "List VM response was not a valid list"
-        )
-        self.assertNotEqual(
-            len(list_vms),
-            0,
-            "List VM response was empty"
-        )
+        self.assertTrue(isinstance(list_vms, list) and len(list_vms) > 0, msg="List VM response empty")
 
         vm = list_vms[0]
         self.assertEqual(
@@ -186,7 +179,6 @@ class TestDeployVM(cloudstackTestCase):
         self.debug(
             "Verify listVirtualMachines response for virtual machines: %s, %s" % (self.virtual_machine.id, self.virtual_machine2.id)
         )
-
         self.assertEqual(
             isinstance(list_vms, list),
             True,
@@ -204,3 +196,225 @@ class TestDeployVM(cloudstackTestCase):
         except Exception as e:
             self.debug("Warning! Exception in tearDown: %s" % e)
 
+class TestDeployVMVolumeCreationFailure(cloudstackTestCase):
+    """Test VM deploy into user account with volume creation failure
+    """
+
+    def setUp(self):
+        self.testdata = TestData().testdata
+        self.apiclient = self.testClient.getApiClient()
+
+        # Get Zone, Domain and Default Built-in template
+        self.domain = get_domain(self.apiclient, self.testdata)
+        self.zone = get_zone(self.apiclient, self.testdata)
+        self.testdata["mode"] = self.zone.networktype
+        self.template = get_template(self.apiclient, self.zone.id, self.testdata["ostype"])
+
+        #create a user account
+        self.account = Account.create(
+            self.apiclient,
+            self.testdata["account"],
+            domainid=self.domain.id
+        )
+        #create a service offering
+        self.service_offering = ServiceOffering.create(
+            self.apiclient,
+            self.testdata["service_offering"]["small"]
+        )
+        #create first VM
+        self.virtual_machine = VirtualMachine.create(
+            self.apiclient,
+            self.testdata["virtual_machine"],
+            accountid=self.account.name,
+            zoneid=self.zone.id,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering.id,
+            templateid=self.template.id)
+        #mock to simulate volume creation failure
+        self.mock_volume_failure = SimulatorMock.create(
+            apiclient=self.apiclient,
+            command="CopyCommand",
+            count=6)
+        #build cleanup list
+        self.cleanup = [
+            self.service_offering,
+            self.account,
+            self.mock_volume_failure
+        ]
+
+    @attr(tags = ['selfservice'])
+    def test_deploy_vm_volume_creation_failure(self):
+        """Test Deploy Virtual Machine - volume creation failure and retry
+
+        # Validate the following:
+        # 1. 1st VM creation failed
+        # 2. Check there were 4 failed volume creation retries (mock count = (6-4) = 2)
+        # 3. 2nd VM creation succeeded
+        # 4. Check there were 2 failed volume creation retries (mock count = (2-2) = 0)
+        # 5. ListVM returns accurate information
+        """
+        self.virtual_machine = None
+        with self.assertRaises(Exception):
+            self.virtual_machine = VirtualMachine.create(
+                self.apiclient,
+                self.testdata["virtual_machine2"],
+                accountid=self.account.name,
+                zoneid=self.zone.id,
+                domainid=self.account.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id)
+
+        self.mock_volume_failure = self.mock_volume_failure.query(self.apiclient)
+        self.assertEqual(
+            self.mock_volume_failure.count,
+            2,
+            msg="Volume failure mock not executed")
+
+        self.virtual_machine = VirtualMachine.create(
+            self.apiclient,
+            self.testdata["virtual_machine3"],
+            accountid=self.account.name,
+            zoneid=self.zone.id,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering.id,
+            templateid=self.template.id)
+        list_vms = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)
+        self.assertTrue(isinstance(list_vms, list) and len(list_vms) > 0, msg="List VM response empty")
+        vm = list_vms[0]
+        self.assertEqual(
+            vm.id,
+            self.virtual_machine.id,
+            "VM ids do not match")
+        self.assertEqual(
+            vm.name,
+            self.virtual_machine.name,
+            "VM names do not match")
+        self.assertEqual(
+            vm.state,
+            "Running",
+            msg="VM is not in Running state")
+
+        self.mock_volume_failure = self.mock_volume_failure.query(self.apiclient)
+        self.assertEqual(
+            self.mock_volume_failure.count,
+            0,
+            msg="Volume failure mock not executed")
+
+    def tearDown(self):
+        try:
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            self.debug("Warning! Exception in tearDown: %s" % e)
+
+
+class TestDeployVMStartFailure(cloudstackTestCase):
+    """Test VM deploy into user account with start operation failure
+    """
+
+    def setUp(self):
+        self.testdata = TestData().testdata
+        self.apiclient = self.testClient.getApiClient()
+
+        # Get Zone, Domain and Default Built-in template
+        self.domain = get_domain(self.apiclient, self.testdata)
+        self.zone = get_zone(self.apiclient, self.testdata)
+        self.testdata["mode"] = self.zone.networktype
+        self.template = get_template(self.apiclient, self.zone.id, self.testdata["ostype"])
+
+        #create a user account
+        self.account = Account.create(
+            self.apiclient,
+            self.testdata["account"],
+            domainid=self.domain.id
+        )
+        #create a service offering
+        self.service_offering = ServiceOffering.create(
+            self.apiclient,
+            self.testdata["service_offering"]["small"]
+        )
+        #create first VM
+        self.virtual_machine = VirtualMachine.create(
+            self.apiclient,
+            self.testdata["virtual_machine"],
+            accountid=self.account.name,
+            zoneid=self.zone.id,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering.id,
+            templateid=self.template.id)
+        #mock to simulate vm start failure
+        self.mock_start_failure = SimulatorMock.create(
+            apiclient=self.apiclient,
+            command="StartCommand",
+            count=6)
+        #build cleanup list
+        self.cleanup = [
+            self.service_offering,
+            self.account,
+            self.mock_start_failure
+        ]
+
+    @attr(tags = ['selfservice'])
+    def test_deploy_vm_start_failure(self):
+        """Test Deploy Virtual Machine - start operation failure and retry
+
+        # Validate the following:
+        # 1. 1st VM creation failed
+        # 2. Check there were 4 failed start operation retries (mock count = (6-4) = 2)
+        # 3. 2nd VM creation succeeded
+        # 4. Check there were 2 failed start operation retries (mock count = (2-2) = 0)
+        # 5. ListVM returns accurate information
+        """
+        self.virtual_machine = None
+        with self.assertRaises(Exception):
+            self.virtual_machine = VirtualMachine.create(
+                self.apiclient,
+                self.testdata["virtual_machine2"],
+                accountid=self.account.name,
+                zoneid=self.zone.id,
+                domainid=self.account.domainid,
+                serviceofferingid=self.service_offering.id,
+                templateid=self.template.id)
+
+        self.mock_start_failure = self.mock_start_failure.query(self.apiclient)
+        self.assertEqual(
+            self.mock_start_failure.count,
+            2,
+            msg="Start failure mock not executed")
+
+        self.virtual_machine = VirtualMachine.create(
+            self.apiclient,
+            self.testdata["virtual_machine3"],
+            accountid=self.account.name,
+            zoneid=self.zone.id,
+            domainid=self.account.domainid,
+            serviceofferingid=self.service_offering.id,
+            templateid=self.template.id)
+        list_vms = VirtualMachine.list(self.apiclient, id=self.virtual_machine.id)
+        self.assertTrue(isinstance(list_vms, list) and len(list_vms) > 0, msg="List VM response empty")
+        vm = list_vms[0]
+        self.assertEqual(
+            vm.id,
+            self.virtual_machine.id,
+            "VM ids do not match")
+        self.assertEqual(
+            vm.name,
+            self.virtual_machine.name,
+            "VM names do not match")
+        self.assertEqual(
+            vm.state,
+            "Running",
+            msg="VM is not in Running state")
+
+        self.mock_start_failure = self.mock_start_failure.query(self.apiclient)
+        self.assertEqual(
+            self.mock_start_failure.count,
+            0,
+            msg="Start failure mock not executed")
+
+    def tearDown(self):
+        try:
+            cleanup_resources(self.apiclient, self.cleanup)
+        except Exception as e:
+            self.debug("Warning! Exception in tearDown: %s" % e)
+
+