You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by al...@apache.org on 2012/06/29 02:45:07 UTC

[33/50] [abbrv] moving out VMWAre and Nexus VSM support code into plugins/hypervisors/vmware

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/8197f1f0/core/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/core/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
deleted file mode 100755
index f4771b8..0000000
--- a/core/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
+++ /dev/null
@@ -1,4201 +0,0 @@
-// Copyright 2012 Citrix Systems, Inc. Licensed under the
-// Apache License, Version 2.0 (the "License"); you may not use this
-// file except in compliance with the License.  Citrix Systems, Inc.
-// reserves all rights not expressly granted by the License.
-// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-// 
-// Automatically generated by addcopyright.py at 04/03/2012
-package com.cloud.hypervisor.vmware.resource;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.ConnectException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.nio.channels.SocketChannel;
-import java.rmi.RemoteException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.TimeZone;
-import java.util.UUID;
-
-import javax.naming.ConfigurationException;
-
-import org.apache.log4j.Logger;
-import org.apache.log4j.NDC;
-
-import com.cloud.agent.IAgentControl;
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.AttachIsoCommand;
-import com.cloud.agent.api.AttachVolumeAnswer;
-import com.cloud.agent.api.AttachVolumeCommand;
-import com.cloud.agent.api.BackupSnapshotAnswer;
-import com.cloud.agent.api.BackupSnapshotCommand;
-import com.cloud.agent.api.BumpUpPriorityCommand;
-import com.cloud.agent.api.CheckHealthAnswer;
-import com.cloud.agent.api.CheckHealthCommand;
-import com.cloud.agent.api.CheckNetworkAnswer;
-import com.cloud.agent.api.CheckNetworkCommand;
-import com.cloud.agent.api.CheckOnHostAnswer;
-import com.cloud.agent.api.CheckOnHostCommand;
-import com.cloud.agent.api.CheckRouterAnswer;
-import com.cloud.agent.api.CheckRouterCommand;
-import com.cloud.agent.api.CheckVirtualMachineAnswer;
-import com.cloud.agent.api.CheckVirtualMachineCommand;
-import com.cloud.agent.api.Command;
-import com.cloud.agent.api.CreatePrivateTemplateFromSnapshotCommand;
-import com.cloud.agent.api.CreatePrivateTemplateFromVolumeCommand;
-import com.cloud.agent.api.CreateStoragePoolCommand;
-import com.cloud.agent.api.CreateVolumeFromSnapshotAnswer;
-import com.cloud.agent.api.CreateVolumeFromSnapshotCommand;
-import com.cloud.agent.api.DeleteStoragePoolCommand;
-import com.cloud.agent.api.GetDomRVersionAnswer;
-import com.cloud.agent.api.GetDomRVersionCmd;
-import com.cloud.agent.api.GetHostStatsAnswer;
-import com.cloud.agent.api.GetHostStatsCommand;
-import com.cloud.agent.api.GetStorageStatsAnswer;
-import com.cloud.agent.api.GetStorageStatsCommand;
-import com.cloud.agent.api.GetVmStatsAnswer;
-import com.cloud.agent.api.GetVmStatsCommand;
-import com.cloud.agent.api.GetVncPortAnswer;
-import com.cloud.agent.api.GetVncPortCommand;
-import com.cloud.agent.api.HostStatsEntry;
-import com.cloud.agent.api.MaintainAnswer;
-import com.cloud.agent.api.MaintainCommand;
-import com.cloud.agent.api.ManageSnapshotAnswer;
-import com.cloud.agent.api.ManageSnapshotCommand;
-import com.cloud.agent.api.MigrateAnswer;
-import com.cloud.agent.api.MigrateCommand;
-import com.cloud.agent.api.ModifySshKeysCommand;
-import com.cloud.agent.api.ModifyStoragePoolAnswer;
-import com.cloud.agent.api.ModifyStoragePoolCommand;
-import com.cloud.agent.api.NetworkUsageAnswer;
-import com.cloud.agent.api.NetworkUsageCommand;
-import com.cloud.agent.api.PingCommand;
-import com.cloud.agent.api.PingRoutingCommand;
-import com.cloud.agent.api.PingTestCommand;
-import com.cloud.agent.api.PoolEjectCommand;
-import com.cloud.agent.api.PrepareForMigrationAnswer;
-import com.cloud.agent.api.PrepareForMigrationCommand;
-import com.cloud.agent.api.ReadyAnswer;
-import com.cloud.agent.api.ReadyCommand;
-import com.cloud.agent.api.RebootAnswer;
-import com.cloud.agent.api.RebootCommand;
-import com.cloud.agent.api.RebootRouterCommand;
-import com.cloud.agent.api.SetupAnswer;
-import com.cloud.agent.api.SetupCommand;
-import com.cloud.agent.api.StartAnswer;
-import com.cloud.agent.api.StartCommand;
-import com.cloud.agent.api.StartupCommand;
-import com.cloud.agent.api.StartupRoutingCommand;
-import com.cloud.agent.api.StartupStorageCommand;
-import com.cloud.agent.api.StopAnswer;
-import com.cloud.agent.api.StopCommand;
-import com.cloud.agent.api.StoragePoolInfo;
-import com.cloud.agent.api.UpgradeSnapshotCommand;
-import com.cloud.agent.api.ValidateSnapshotAnswer;
-import com.cloud.agent.api.ValidateSnapshotCommand;
-import com.cloud.agent.api.VmStatsEntry;
-import com.cloud.agent.api.check.CheckSshAnswer;
-import com.cloud.agent.api.check.CheckSshCommand;
-import com.cloud.agent.api.routing.DhcpEntryCommand;
-import com.cloud.agent.api.routing.IpAssocAnswer;
-import com.cloud.agent.api.routing.IpAssocCommand;
-import com.cloud.agent.api.routing.LoadBalancerConfigCommand;
-import com.cloud.agent.api.routing.NetworkElementCommand;
-import com.cloud.agent.api.routing.RemoteAccessVpnCfgCommand;
-import com.cloud.agent.api.routing.SavePasswordCommand;
-import com.cloud.agent.api.routing.SetFirewallRulesAnswer;
-import com.cloud.agent.api.routing.SetFirewallRulesCommand;
-import com.cloud.agent.api.routing.SetPortForwardingRulesAnswer;
-import com.cloud.agent.api.routing.SetPortForwardingRulesCommand;
-import com.cloud.agent.api.routing.SetStaticNatRulesAnswer;
-import com.cloud.agent.api.routing.SetStaticNatRulesCommand;
-import com.cloud.agent.api.routing.VmDataCommand;
-import com.cloud.agent.api.routing.VpnUsersCfgCommand;
-import com.cloud.agent.api.storage.CopyVolumeAnswer;
-import com.cloud.agent.api.storage.CopyVolumeCommand;
-import com.cloud.agent.api.storage.CreateAnswer;
-import com.cloud.agent.api.storage.CreateCommand;
-import com.cloud.agent.api.storage.CreatePrivateTemplateAnswer;
-import com.cloud.agent.api.storage.DestroyCommand;
-import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer;
-import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand;
-import com.cloud.agent.api.to.IpAddressTO;
-import com.cloud.agent.api.to.NicTO;
-import com.cloud.agent.api.to.PortForwardingRuleTO;
-import com.cloud.agent.api.to.StaticNatRuleTO;
-import com.cloud.agent.api.to.StorageFilerTO;
-import com.cloud.agent.api.to.VirtualMachineTO;
-import com.cloud.agent.api.to.VolumeTO;
-import com.cloud.dc.DataCenter.NetworkType;
-import com.cloud.dc.Vlan;
-import com.cloud.exception.InternalErrorException;
-import com.cloud.host.Host.Type;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.hypervisor.vmware.manager.VmwareHostService;
-import com.cloud.hypervisor.vmware.manager.VmwareManager;
-import com.cloud.hypervisor.vmware.mo.ClusterMO;
-import com.cloud.hypervisor.vmware.mo.CustomFieldConstants;
-import com.cloud.hypervisor.vmware.mo.CustomFieldsManagerMO;
-import com.cloud.hypervisor.vmware.mo.DatacenterMO;
-import com.cloud.hypervisor.vmware.mo.DatastoreMO;
-import com.cloud.hypervisor.vmware.mo.DiskControllerType;
-import com.cloud.hypervisor.vmware.mo.HostFirewallSystemMO;
-import com.cloud.hypervisor.vmware.mo.HostMO;
-import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper;
-import com.cloud.hypervisor.vmware.mo.NetworkDetails;
-import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType;
-import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
-import com.cloud.hypervisor.vmware.mo.VirtualSwitchType;
-import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost;
-import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostNetworkSummary;
-import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostResourceSummary;
-import com.cloud.hypervisor.vmware.util.VmwareContext;
-import com.cloud.hypervisor.vmware.util.VmwareGuestOsMapper;
-import com.cloud.hypervisor.vmware.util.VmwareHelper;
-import com.cloud.network.HAProxyConfigurator;
-import com.cloud.network.LoadBalancerConfigurator;
-import com.cloud.network.Networks;
-import com.cloud.network.Networks.BroadcastDomainType;
-import com.cloud.resource.ServerResource;
-import com.cloud.serializer.GsonHelper;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.StoragePoolType;
-import com.cloud.storage.Volume;
-import com.cloud.storage.resource.StoragePoolResource;
-import com.cloud.storage.template.TemplateInfo;
-import com.cloud.utils.DateUtil;
-import com.cloud.utils.Pair;
-import com.cloud.utils.StringUtils;
-import com.cloud.utils.component.ComponentLocator;
-import com.cloud.utils.db.DB;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.exception.ExceptionUtil;
-import com.cloud.utils.mgmt.JmxUtil;
-import com.cloud.utils.mgmt.PropertyMapDynamicBean;
-import com.cloud.utils.net.NetUtils;
-import com.cloud.vm.DiskProfile;
-import com.cloud.vm.VirtualMachine;
-import com.cloud.vm.VirtualMachine.State;
-import com.cloud.vm.VirtualMachineName;
-import com.cloud.vm.VmDetailConstants;
-import com.google.gson.Gson;
-import com.vmware.vim25.AboutInfo;
-import com.vmware.vim25.ClusterDasConfigInfo;
-import com.vmware.vim25.ComputeResourceSummary;
-import com.vmware.vim25.DatastoreSummary;
-import com.vmware.vim25.DynamicProperty;
-import com.vmware.vim25.HostFirewallInfo;
-import com.vmware.vim25.HostFirewallRuleset;
-import com.vmware.vim25.HostNetworkTrafficShapingPolicy;
-import com.vmware.vim25.HostPortGroupSpec;
-import com.vmware.vim25.ManagedObjectReference;
-import com.vmware.vim25.ObjectContent;
-import com.vmware.vim25.OptionValue;
-import com.vmware.vim25.PerfCounterInfo;
-import com.vmware.vim25.PerfEntityMetric;
-import com.vmware.vim25.PerfEntityMetricBase;
-import com.vmware.vim25.PerfMetricId;
-import com.vmware.vim25.PerfMetricIntSeries;
-import com.vmware.vim25.PerfMetricSeries;
-import com.vmware.vim25.PerfQuerySpec;
-import com.vmware.vim25.PerfSampleInfo;
-import com.vmware.vim25.RuntimeFault;
-import com.vmware.vim25.ToolsUnavailable;
-import com.vmware.vim25.VimPortType;
-import com.vmware.vim25.VirtualDevice;
-import com.vmware.vim25.VirtualDeviceConfigSpec;
-import com.vmware.vim25.VirtualDeviceConfigSpecOperation;
-import com.vmware.vim25.VirtualDisk;
-import com.vmware.vim25.VirtualEthernetCard;
-import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo;
-import com.vmware.vim25.VirtualLsiLogicController;
-import com.vmware.vim25.VirtualMachineConfigSpec;
-import com.vmware.vim25.VirtualMachineFileInfo;
-import com.vmware.vim25.VirtualMachineGuestOsIdentifier;
-import com.vmware.vim25.VirtualMachinePowerState;
-import com.vmware.vim25.VirtualMachineRuntimeInfo;
-import com.vmware.vim25.VirtualSCSISharing;
-
-public class VmwareResource implements StoragePoolResource, ServerResource, VmwareHostService {
-    private static final Logger s_logger = Logger.getLogger(VmwareResource.class);
-
-    protected String _name;
-
-    protected final long _ops_timeout = 900000; 		// 15 minutes time out to time
-    protected final int _shutdown_waitMs = 300000;		// wait up to 5 minutes for shutdown 
-    
-    // out an operation
-    protected final int _retry = 24;
-    protected final int _sleep = 10000;
-    protected final int DEFAULT_DOMR_SSHPORT = 3922;
-    protected final int MAX_CMD_MBEAN = 100;
-
-    protected String _url;
-    protected String _dcId;
-    protected String _pod;
-    protected String _cluster;
-    protected String _username;
-    protected String _password;
-    protected String _guid;
-    protected String _vCenterAddress;
-
-    protected String _privateNetworkVSwitchName;
-    protected String _publicNetworkVSwitchName;
-    protected String _guestNetworkVSwitchName;
-    protected VirtualSwitchType _vSwitchType = VirtualSwitchType.StandardVirtualSwitch;
-    protected boolean _nexusVSwitch = false;
-    
-    protected float _cpuOverprovisioningFactor = 1;
-    protected boolean _reserveCpu = false;
-    
-    protected float _memOverprovisioningFactor = 1;
-    protected boolean _reserveMem = false;
-    protected boolean _recycleHungWorker = false;
-    protected DiskControllerType _rootDiskController = DiskControllerType.ide;    
-
-    protected ManagedObjectReference _morHyperHost;
-    protected VmwareContext _serviceContext;
-    protected String _hostName;
-
-    protected HashMap<String, State> _vms = new HashMap<String, State>(71);
-    protected List<PropertyMapDynamicBean> _cmdMBeans = new ArrayList<PropertyMapDynamicBean>();
-
-    protected Gson _gson;
-
-    protected volatile long _cmdSequence = 1;
-
-    protected static HashMap<VirtualMachinePowerState, State> s_statesTable;
-    static {
-        s_statesTable = new HashMap<VirtualMachinePowerState, State>();
-        s_statesTable.put(VirtualMachinePowerState.poweredOn, State.Running);
-        s_statesTable.put(VirtualMachinePowerState.poweredOff, State.Stopped);
-        s_statesTable.put(VirtualMachinePowerState.suspended, State.Stopped);
-    }
-
-    public VmwareResource() {
-        _gson = GsonHelper.getGsonLogger();
-    }
-
-    @Override
-    public Answer executeRequest(Command cmd) {
-    	if(s_logger.isTraceEnabled())
-    		s_logger.trace("Begin executeRequest(), cmd: " + cmd.getClass().getSimpleName());
-    	
-        Answer answer = null;
-        NDC.push(_hostName != null ? _hostName : _guid + "(" + ComponentLocator.class.getPackage().getImplementationVersion() + ")");
-        try {
-            long cmdSequence = _cmdSequence++;
-            Date startTime = DateUtil.currentGMTTime();
-            PropertyMapDynamicBean mbean = new PropertyMapDynamicBean();
-            mbean.addProp("StartTime", DateUtil.getDateDisplayString(TimeZone.getDefault(), startTime));
-            mbean.addProp("Command", _gson.toJson(cmd));
-            mbean.addProp("Sequence", String.valueOf(cmdSequence));
-            mbean.addProp("Name", cmd.getClass().getSimpleName());
-
-            if (cmd instanceof CreateCommand) {
-                answer = execute((CreateCommand) cmd);
-            } else if (cmd instanceof SetPortForwardingRulesCommand) {
-                answer = execute((SetPortForwardingRulesCommand) cmd);
-            } else if (cmd instanceof SetStaticNatRulesCommand) {
-                answer = execute((SetStaticNatRulesCommand) cmd);
-            } else if (cmd instanceof LoadBalancerConfigCommand) {
-                answer = execute((LoadBalancerConfigCommand) cmd);
-            } else if (cmd instanceof IpAssocCommand) {
-                answer = execute((IpAssocCommand) cmd);
-            } else if (cmd instanceof SavePasswordCommand) {
-                answer = execute((SavePasswordCommand) cmd);
-            } else if (cmd instanceof DhcpEntryCommand) {
-                answer = execute((DhcpEntryCommand) cmd);
-            } else if (cmd instanceof VmDataCommand) {
-                answer = execute((VmDataCommand) cmd);
-            } else if (cmd instanceof ReadyCommand) {
-                answer = execute((ReadyCommand) cmd);
-            } else if (cmd instanceof GetHostStatsCommand) {
-                answer = execute((GetHostStatsCommand) cmd);
-            } else if (cmd instanceof GetVmStatsCommand) {
-                answer = execute((GetVmStatsCommand) cmd);
-            } else if (cmd instanceof CheckHealthCommand) {
-                answer = execute((CheckHealthCommand) cmd);
-            } else if (cmd instanceof StopCommand) {
-                answer = execute((StopCommand) cmd);
-            } else if (cmd instanceof RebootRouterCommand) {
-                answer = execute((RebootRouterCommand) cmd);
-            } else if (cmd instanceof RebootCommand) {
-                answer = execute((RebootCommand) cmd);
-            } else if (cmd instanceof CheckVirtualMachineCommand) {
-                answer = execute((CheckVirtualMachineCommand) cmd);
-            } else if (cmd instanceof PrepareForMigrationCommand) {
-                answer = execute((PrepareForMigrationCommand) cmd);
-            } else if (cmd instanceof MigrateCommand) {
-                answer = execute((MigrateCommand) cmd);
-            } else if (cmd instanceof DestroyCommand) {
-                answer = execute((DestroyCommand) cmd);
-            } else if (cmd instanceof CreateStoragePoolCommand) {
-                return execute((CreateStoragePoolCommand) cmd);
-            } else if (cmd instanceof ModifyStoragePoolCommand) {
-                answer = execute((ModifyStoragePoolCommand) cmd);
-            } else if (cmd instanceof DeleteStoragePoolCommand) {
-                answer = execute((DeleteStoragePoolCommand) cmd);
-            } else if (cmd instanceof CopyVolumeCommand) {
-                answer = execute((CopyVolumeCommand) cmd);
-            } else if (cmd instanceof AttachVolumeCommand) {
-                answer = execute((AttachVolumeCommand) cmd);
-            } else if (cmd instanceof AttachIsoCommand) {
-                answer = execute((AttachIsoCommand) cmd);
-            } else if (cmd instanceof ValidateSnapshotCommand) {
-                answer = execute((ValidateSnapshotCommand) cmd);
-            } else if (cmd instanceof ManageSnapshotCommand) {
-                answer = execute((ManageSnapshotCommand) cmd);
-            } else if (cmd instanceof BackupSnapshotCommand) {
-                answer = execute((BackupSnapshotCommand) cmd);
-            } else if (cmd instanceof CreateVolumeFromSnapshotCommand) {
-                answer = execute((CreateVolumeFromSnapshotCommand) cmd);
-            } else if (cmd instanceof CreatePrivateTemplateFromVolumeCommand) {
-                answer = execute((CreatePrivateTemplateFromVolumeCommand) cmd);
-            } else if (cmd instanceof CreatePrivateTemplateFromSnapshotCommand) {
-                answer = execute((CreatePrivateTemplateFromSnapshotCommand) cmd);
-            } else if (cmd instanceof UpgradeSnapshotCommand) {
-                answer = execute((UpgradeSnapshotCommand) cmd);
-            } else if (cmd instanceof GetStorageStatsCommand) {
-                answer = execute((GetStorageStatsCommand) cmd);
-            } else if (cmd instanceof PrimaryStorageDownloadCommand) {
-                answer = execute((PrimaryStorageDownloadCommand) cmd);
-            } else if (cmd instanceof GetVncPortCommand) {
-                answer = execute((GetVncPortCommand) cmd);
-            } else if (cmd instanceof SetupCommand) {
-                answer = execute((SetupCommand) cmd);
-            } else if (cmd instanceof MaintainCommand) {
-                answer = execute((MaintainCommand) cmd);
-            } else if (cmd instanceof PingTestCommand) {
-                answer = execute((PingTestCommand) cmd);
-            } else if (cmd instanceof CheckOnHostCommand) {
-                answer = execute((CheckOnHostCommand) cmd);
-            } else if (cmd instanceof ModifySshKeysCommand) {
-                answer = execute((ModifySshKeysCommand) cmd);
-            } else if (cmd instanceof PoolEjectCommand) {
-                answer = execute((PoolEjectCommand) cmd);
-            } else if (cmd instanceof NetworkUsageCommand) {
-                answer = execute((NetworkUsageCommand) cmd);
-            } else if (cmd instanceof StartCommand) {
-                answer = execute((StartCommand) cmd);
-            } else if (cmd instanceof RemoteAccessVpnCfgCommand) {
-                answer = execute((RemoteAccessVpnCfgCommand) cmd);
-            } else if (cmd instanceof VpnUsersCfgCommand) {
-                answer = execute((VpnUsersCfgCommand) cmd);
-            } else if (cmd instanceof CheckSshCommand) {
-                answer = execute((CheckSshCommand) cmd);
-            } else if (cmd instanceof CheckRouterCommand) {
-                answer = execute((CheckRouterCommand) cmd);
-            } else  if (cmd instanceof SetFirewallRulesCommand) {
-            	answer = execute((SetFirewallRulesCommand)cmd);
-            } else if (cmd instanceof BumpUpPriorityCommand) {
-                answer = execute((BumpUpPriorityCommand)cmd);
-            } else if (cmd instanceof GetDomRVersionCmd) {
-                answer = execute((GetDomRVersionCmd)cmd);
-            } else if (cmd instanceof CheckNetworkCommand) {
-                answer = execute((CheckNetworkCommand) cmd);
-            } else {
-                answer = Answer.createUnsupportedCommandAnswer(cmd);
-            }
-
-            if(cmd.getContextParam("checkpoint") != null) {
-                answer.setContextParam("checkpoint", cmd.getContextParam("checkpoint"));
-            }
-
-            Date doneTime = DateUtil.currentGMTTime();
-            mbean.addProp("DoneTime", DateUtil.getDateDisplayString(TimeZone.getDefault(), doneTime));
-            mbean.addProp("Answer", _gson.toJson(answer));
-
-            synchronized (this) {
-                try {
-                    JmxUtil.registerMBean("VMware " + _morHyperHost.get_value(), "Command " + cmdSequence + "-" + cmd.getClass().getSimpleName(), mbean);
-                    _cmdMBeans.add(mbean);
-
-                    if (_cmdMBeans.size() >= MAX_CMD_MBEAN) {
-                        PropertyMapDynamicBean mbeanToRemove = _cmdMBeans.get(0);
-                        _cmdMBeans.remove(0);
-
-                        JmxUtil.unregisterMBean("VMware " + _morHyperHost.get_value(), "Command " + mbeanToRemove.getProp("Sequence") + "-" + mbeanToRemove.getProp("Name"));
-                    }
-                } catch (Exception e) {
-                	if(s_logger.isTraceEnabled())
-                		s_logger.trace("Unable to register JMX monitoring due to exception " + ExceptionUtil.toString(e));
-                }
-            }
-
-        } finally {
-            NDC.pop();
-        }
-
-    	if(s_logger.isTraceEnabled())
-    		s_logger.trace("End executeRequest(), cmd: " + cmd.getClass().getSimpleName());
-        
-        return answer;
-    }
-    
-    protected Answer execute(CheckNetworkCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource CheckNetworkCommand " + _gson.toJson(cmd));
-        }
-
-        // TODO setup portgroup for private network needs to be done here now
-        return new CheckNetworkAnswer(cmd, true , "Network Setup check by names is done");
-    }
-    
-    protected Answer execute(NetworkUsageCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource NetworkUsageCommand " + _gson.toJson(cmd));
-        }
-        if(cmd.getOption()!=null && cmd.getOption().equals("create") ){
-            String result = networkUsage(cmd.getPrivateIP(), "create", null);
-            NetworkUsageAnswer answer = new NetworkUsageAnswer(cmd, result, 0L, 0L);
-            return answer;
-        }
-        long[] stats = getNetworkStats(cmd.getPrivateIP());
-
-        NetworkUsageAnswer answer = new NetworkUsageAnswer(cmd, "", stats[0], stats[1]);
-        return answer;
-    }
-
-    protected Answer execute(SetPortForwardingRulesCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource SetPortForwardingRulesCommand: " + _gson.toJson(cmd));
-        }
-
-        String controlIp = getRouterSshControlIp(cmd);
-        String args = "";
-        String[] results = new String[cmd.getRules().length];
-        int i = 0;
-        
-        boolean endResult = true;
-        for (PortForwardingRuleTO rule : cmd.getRules()) {
-            args += rule.revoked() ? " -D " : " -A ";
-            args += " -P " + rule.getProtocol().toLowerCase();
-            args += " -l " + rule.getSrcIp();
-            args += " -p " + rule.getStringSrcPortRange();
-            args += " -r " + rule.getDstIp();
-            args += " -d " + rule.getStringDstPortRange();
-
-            try {
-                VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-                Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/root/firewall.sh " + args);
-
-                if (s_logger.isDebugEnabled())
-                    s_logger.debug("Executing script on domain router " + controlIp + ": /root/firewall.sh " + args);
-
-                if (!result.first()) {
-                    s_logger.error("SetPortForwardingRulesCommand failure on setting one rule. args: " + args);
-                    results[i++] = "Failed";
-                    endResult = false;
-                } else {
-                    results[i++] = null;
-                }
-            } catch (Throwable e) {
-                s_logger.error("SetPortForwardingRulesCommand(args: " + args + ") failed on setting one rule due to " + VmwareHelper.getExceptionMessage(e), e);
-                results[i++] = "Failed";
-                endResult = false;
-            }
-        }
-
-        return new SetPortForwardingRulesAnswer(cmd, results, endResult);
-    }
-    
-    protected SetFirewallRulesAnswer execute(SetFirewallRulesCommand cmd) {
-		String controlIp = getRouterSshControlIp(cmd);
-		String[] results = new String[cmd.getRules().length];
-
-		String[][] rules = cmd.generateFwRules();
-		String args = "";
-		args += " -F ";
-		StringBuilder sb = new StringBuilder();
-		String[] fwRules = rules[0];
-		if (fwRules.length > 0) {
-			for (int i = 0; i < fwRules.length; i++) {
-				sb.append(fwRules[i]).append(',');
-			}
-			args += " -a " + sb.toString();
-		}
-
-		try {
-			VmwareManager mgr = getServiceContext().getStockObject(
-					VmwareManager.CONTEXT_STOCK_NAME);
-			Pair<Boolean, String> result = SshHelper.sshExecute(controlIp,
-					DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(),
-					null, "/root/firewall_rule.sh " + args);
-
-			if (s_logger.isDebugEnabled())
-				s_logger.debug("Executing script on domain router " + controlIp
-						+ ": /root/firewall_rule.sh " + args);
-
-			if (!result.first()) {
-				s_logger.error("SetFirewallRulesCommand failure on setting one rule. args: "
-						+ args);
-				//FIXME - in the future we have to process each rule separately; now we temporarily set every rule to be false if single rule fails
-	            for (int i=0; i < results.length; i++) {
-	                results[i] = "Failed";
-	            }
-	            
-	            return new SetFirewallRulesAnswer(cmd, false, results);
-			} 
-		} catch (Throwable e) {
-			s_logger.error("SetFirewallRulesCommand(args: " + args
-					+ ") failed on setting one rule due to "
-					+ VmwareHelper.getExceptionMessage(e), e);
-			//FIXME - in the future we have to process each rule separately; now we temporarily set every rule to be false if single rule fails
-            for (int i=0; i < results.length; i++) {
-                results[i] = "Failed";
-            }
-			return new SetFirewallRulesAnswer(cmd, false, results);
-		} 
-
-		return new SetFirewallRulesAnswer(cmd, true, results);
-    }	
-    
-    protected Answer execute(SetStaticNatRulesCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource SetFirewallRuleCommand: " + _gson.toJson(cmd));
-        }
-
-        String args = null;
-        String[] results = new String[cmd.getRules().length];
-        int i = 0;
-        boolean endResult = true;
-        for (StaticNatRuleTO rule : cmd.getRules()) {
-            // 1:1 NAT needs instanceip;publicip;domrip;op
-            args = rule.revoked() ? " -D " : " -A ";
-
-            args += " -l " + rule.getSrcIp();
-            args += " -r " + rule.getDstIp();
-            
-            if (rule.getProtocol() != null) {
-                args += " -P " + rule.getProtocol().toLowerCase();
-            }
-            
-            args += " -d " + rule.getStringSrcPortRange();
-            args += " -G ";
-
-            try {
-                VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-                String controlIp = getRouterSshControlIp(cmd);
-                Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/root/firewall.sh " + args);
-
-                if (s_logger.isDebugEnabled())
-                    s_logger.debug("Executing script on domain router " + controlIp + ": /root/firewall.sh " + args);
-
-                if (!result.first()) {
-                    s_logger.error("SetStaticNatRulesCommand failure on setting one rule. args: " + args);
-                    results[i++] = "Failed";
-                    endResult = false;
-                } else {
-                    results[i++] = null;
-                }
-            } catch (Throwable e) {
-                s_logger.error("SetStaticNatRulesCommand (args: " + args + ") failed on setting one rule due to " + VmwareHelper.getExceptionMessage(e), e);
-                results[i++] = "Failed";
-                endResult = false;
-            }
-        }
-        return new SetStaticNatRulesAnswer(cmd, results, endResult);
-    }
-
-    protected Answer execute(final LoadBalancerConfigCommand cmd) {
-        VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-        File keyFile = mgr.getSystemVMKeyFile();
-
-        String routerIp = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
-        String controlIp = getRouterSshControlIp(cmd);
-        
-        assert(controlIp != null);
-
-        LoadBalancerConfigurator cfgtr = new HAProxyConfigurator();
-        String[] config = cfgtr.generateConfiguration(cmd);
-
-        String[][] rules = cfgtr.generateFwRules(cmd);
-        String tmpCfgFilePath = "/tmp/" + routerIp.replace('.', '_') + ".cfg";
-        String tmpCfgFileContents = "";
-        for (int i = 0; i < config.length; i++) {
-            tmpCfgFileContents += config[i];
-            tmpCfgFileContents += "\n";
-        }
-
-        try {
-            SshHelper.scpTo(controlIp, DEFAULT_DOMR_SSHPORT, "root", keyFile, null, "/tmp/", tmpCfgFileContents.getBytes(), routerIp.replace('.', '_') + ".cfg", null);
-
-            try {
-                String[] addRules = rules[LoadBalancerConfigurator.ADD];
-                String[] removeRules = rules[LoadBalancerConfigurator.REMOVE];
-                String[] statRules = rules[LoadBalancerConfigurator.STATS];
-
-                String args = "";
-                args += "-i " + routerIp;
-                args += " -f " + tmpCfgFilePath;
-
-                StringBuilder sb = new StringBuilder();
-                if (addRules.length > 0) {
-                    for (int i = 0; i < addRules.length; i++) {
-                        sb.append(addRules[i]).append(',');
-                    }
-
-                    args += " -a " + sb.toString();
-                }
-
-                sb = new StringBuilder();
-                if (removeRules.length > 0) {
-                    for (int i = 0; i < removeRules.length; i++) {
-                        sb.append(removeRules[i]).append(',');
-                    }
-
-                    args += " -d " + sb.toString();
-                }
-
-                sb = new StringBuilder();
-                if (statRules.length > 0) {
-                    for (int i = 0; i < statRules.length; i++) {
-                        sb.append(statRules[i]).append(',');
-                    }
-
-                    args += " -s " + sb.toString();
-                }
-                
-                Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "scp " + tmpCfgFilePath + " /etc/haproxy/haproxy.cfg.new");
-
-                if (!result.first()) {
-                    s_logger.error("Unable to copy haproxy configuration file");
-                    return new Answer(cmd, false, "LoadBalancerConfigCommand failed due to uanble to copy haproxy configuration file");
-                }
-
-                if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Run command on domain router " + routerIp + ",  /root/loadbalancer.sh " + args);
-                }
-
-                result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/root/loadbalancer.sh " + args);
-
-                if (!result.first()) {
-                    String msg = "LoadBalancerConfigCommand on domain router " + routerIp + " failed. message: " + result.second();
-                    s_logger.error(msg);
-
-                    return new Answer(cmd, false, msg);
-                }
-
-                if (s_logger.isInfoEnabled()) {
-                    s_logger.info("LoadBalancerConfigCommand on domain router " + routerIp + " completed");
-                }
-            } finally {
-                SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "rm " + tmpCfgFilePath);
-            }
-
-            return new Answer(cmd);
-        } catch (Throwable e) {
-            s_logger.error("Unexpected exception: " + e.toString(), e);
-            return new Answer(cmd, false, "LoadBalancerConfigCommand failed due to " + VmwareHelper.getExceptionMessage(e));
-        }
-    }
-
-    protected void assignPublicIpAddress(VirtualMachineMO vmMo, final String vmName, final String privateIpAddress, final String publicIpAddress, final boolean add, final boolean firstIP,
-            final boolean sourceNat, final String vlanId, final String vlanGateway, final String vlanNetmask, final String vifMacAddress, String guestIp) throws Exception {
-
-        String publicNeworkName = HypervisorHostHelper.getPublicNetworkNamePrefix(vlanId);
-        Pair<Integer, VirtualDevice> publicNicInfo = vmMo.getNicDeviceIndex(publicNeworkName);
-
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Find public NIC index, public network name: " + publicNeworkName + ", index: " + publicNicInfo.first());
-        }
-
-        boolean addVif = false;
-        boolean removeVif = false;
-        if (add && publicNicInfo.first().intValue() == -1) {
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("Plug new NIC to associate" + privateIpAddress + " to " + publicIpAddress);
-            }
-
-            addVif = true;
-        } else if (!add && firstIP) {
-            removeVif = true;
-
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("Unplug NIC " + publicNicInfo.first());
-            }
-        }
-
-        if (addVif) {
-            plugPublicNic(vmMo, vlanId, vifMacAddress);
-            publicNicInfo = vmMo.getNicDeviceIndex(publicNeworkName);
-            if (publicNicInfo.first().intValue() >= 0) {
-                networkUsage(privateIpAddress, "addVif", "eth" + publicNicInfo.first());
-            }
-        }
-
-        if (publicNicInfo.first().intValue() < 0) {
-            String msg = "Failed to find DomR VIF to associate/disassociate IP with.";
-            s_logger.error(msg);
-            throw new InternalErrorException(msg);
-        }
-
-        String args = null;
-
-        if (add) {
-            args = " -A ";
-        } else {
-            args = " -D ";
-        }
-
-        if (sourceNat) {
-            args += " -s ";
-        }
-        if (firstIP) {
-            args += " -f ";
-        }
-        String cidrSize = Long.toString(NetUtils.getCidrSize(vlanNetmask));
-        args += " -l ";
-        args += publicIpAddress + "/" + cidrSize;
-
-        args += " -c ";
-        args += "eth" + publicNicInfo.first();
-        
-        args += " -g ";
-        args += vlanGateway;
-
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Run command on domain router " + privateIpAddress + ", /root/ipassoc.sh " + args);
-        }
-
-        VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-        Pair<Boolean, String> result = SshHelper.sshExecute(privateIpAddress, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/root/ipassoc.sh " + args);
-
-        if (!result.first()) {
-            s_logger.error("ipassoc command on domain router " + privateIpAddress + " failed. message: " + result.second());
-            throw new Exception("ipassoc failed due to " + result.second());
-        }
-
-        if (removeVif) {
-        	
-        	String nicMasksStr = vmMo.getCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK);
-        	int nicMasks = Integer.parseInt(nicMasksStr);
-        	nicMasks &= ~(1 << publicNicInfo.first().intValue());
-        	vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMasks));
-
-            HostMO hostMo = vmMo.getRunningHost();
-            List<NetworkDetails> networks = vmMo.getNetworksWithDetails();
-            for (NetworkDetails netDetails : networks) {
-                if (netDetails.getGCTag() != null && netDetails.getGCTag().equalsIgnoreCase("true")) {
-                    if (netDetails.getVMMorsOnNetwork() == null || netDetails.getVMMorsOnNetwork().length == 1) {
-                        cleanupNetwork(hostMo, netDetails);
-                    }
-                }
-            }
-        }
-
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("ipassoc command on domain router " + privateIpAddress + " completed");
-        }
-    }
-
-    private void plugPublicNic(VirtualMachineMO vmMo, final String vlanId, final String vifMacAddress) throws Exception {
-        // TODO : probably need to set traffic shaping
-        Pair<ManagedObjectReference, String> networkInfo = null;
-        
-        if (!_nexusVSwitch) {
-            networkInfo = HypervisorHostHelper.prepareNetwork(this._publicNetworkVSwitchName, "cloud.public",
-                    vmMo.getRunningHost(), vlanId, null, null, this._ops_timeout, true);
-        } else {
-            networkInfo = HypervisorHostHelper.prepareNetwork(this._publicNetworkVSwitchName, "cloud.public",
-                    vmMo.getRunningHost(), vlanId, null, null, this._ops_timeout);
-        }
-
-        int nicIndex = allocPublicNicIndex(vmMo);
-
-        try {
-            VirtualDevice[] nicDevices = vmMo.getNicDevices();
-
-            VirtualEthernetCard device = (VirtualEthernetCard) nicDevices[nicIndex];
-
-            if (!_nexusVSwitch) {
-                VirtualEthernetCardNetworkBackingInfo nicBacking = new VirtualEthernetCardNetworkBackingInfo();
-                nicBacking.setDeviceName(networkInfo.second());
-                nicBacking.setNetwork(networkInfo.first());
-                device.setBacking(nicBacking);
-            } else {
-                HostMO hostMo = vmMo.getRunningHost();
-                DatacenterMO dataCenterMo = new DatacenterMO(hostMo.getContext(), hostMo.getHyperHostDatacenter());
-                device.setBacking(dataCenterMo.getDvPortBackingInfo(networkInfo));
-            }
-
-            VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
-            VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[1];
-            deviceConfigSpecArray[0] = new VirtualDeviceConfigSpec();
-            deviceConfigSpecArray[0].setDevice(device);
-            deviceConfigSpecArray[0].setOperation(VirtualDeviceConfigSpecOperation.edit);
-            
-            vmConfigSpec.setDeviceChange(deviceConfigSpecArray);
-            if(!vmMo.configureVm(vmConfigSpec)) {
-                throw new Exception("Failed to configure devices when plugPublicNic");
-            }
-        } catch(Exception e) {
-        
-        	// restore allocation mask in case of exceptions
-        	String nicMasksStr = vmMo.getCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK);
-        	int nicMasks = Integer.parseInt(nicMasksStr);
-        	nicMasks &= ~(1 << nicIndex);
-        	vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMasks));
-        	
-        	throw e;
-        }
-    }
-    
-    private int allocPublicNicIndex(VirtualMachineMO vmMo) throws Exception {
-    	String nicMasksStr = vmMo.getCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK);
-    	if(nicMasksStr == null || nicMasksStr.isEmpty()) {
-    		throw new Exception("Could not find NIC allocation info");
-    	}
-    	
-    	int nicMasks = Integer.parseInt(nicMasksStr);
-    	VirtualDevice[] nicDevices = vmMo.getNicDevices();
-    	for(int i = 3; i < nicDevices.length; i++) {
-    		if((nicMasks & (1 << i)) == 0) {
-    			nicMasks |= (1 << i);
-    			vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMasks));
-    			return i;
-    		}
-    	}
-    	
-    	throw new Exception("Could not allocate a free public NIC");
-    }
-
-    protected Answer execute(IpAssocCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource IPAssocCommand: " + _gson.toJson(cmd));
-        }
-
-        int i = 0;
-        String[] results = new String[cmd.getIpAddresses().length];
-
-        VmwareContext context = getServiceContext();
-        try {
-            VmwareHypervisorHost hyperHost = getHyperHost(context);
-
-            IpAddressTO[] ips = cmd.getIpAddresses();
-            String routerName = cmd.getAccessDetail(NetworkElementCommand.ROUTER_NAME);
-            String controlIp = VmwareResource.getRouterSshControlIp(cmd);
-
-            VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(routerName);
-
-            // command may sometimes be redirect to a wrong host, we relax
-            // the check and will try to find it within cluster
-            if(vmMo == null) {
-                if(hyperHost instanceof HostMO) {
-                    ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(),
-                        ((HostMO)hyperHost).getParentMor());
-                    vmMo = clusterMo.findVmOnHyperHost(routerName);
-                }
-            }
-
-            if (vmMo == null) {
-                String msg = "Router " + routerName + " no longer exists to execute IPAssoc command";
-                s_logger.error(msg);
-                throw new Exception(msg);
-            }
-
-            for (IpAddressTO ip : ips) {
-                assignPublicIpAddress(vmMo, routerName, controlIp, ip.getPublicIp(), ip.isAdd(), ip.isFirstIP(), ip.isSourceNat(), ip.getVlanId(), ip.getVlanGateway(), ip.getVlanNetmask(),
-                        ip.getVifMacAddress(), ip.getGuestIp());
-                results[i++] = ip.getPublicIp() + " - success";
-            }
-        } catch (Throwable e) {
-            s_logger.error("Unexpected exception: " + e.toString() + " will shortcut rest of IPAssoc commands", e);
-
-            for (; i < cmd.getIpAddresses().length; i++) {
-                results[i++] = IpAssocAnswer.errorResult;
-            }
-        }
-
-        return new IpAssocAnswer(cmd, results);
-    }
-
-    protected Answer execute(SavePasswordCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource SavePasswordCommand. vmName: " + cmd.getVmName() + ", vmIp: " + cmd.getVmIpAddress() + ", password: " 
-            	+ StringUtils.getMaskedPasswordForDisplay(cmd.getPassword()));
-        }
-
-        String controlIp = getRouterSshControlIp(cmd);
-        final String password = cmd.getPassword();
-        final String vmIpAddress = cmd.getVmIpAddress();
-
-        // Run save_password_to_domr.sh
-        String args = " -v " + vmIpAddress;
-
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Run command on domain router " + controlIp + ", /root/savepassword.sh " + args + " -p " + StringUtils.getMaskedPasswordForDisplay(cmd.getPassword()));
-        }
-        
-        args += " -p " + password;
-
-
-        try {
-            VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-            Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/root/savepassword.sh " + args);
-
-            if (!result.first()) {
-                s_logger.error("savepassword command on domain router " + controlIp + " failed, message: " + result.second());
-
-                return new Answer(cmd, false, "SavePassword failed due to " + result.second());
-            }
-
-            if (s_logger.isInfoEnabled()) {
-                s_logger.info("savepassword command on domain router " + controlIp + " completed");
-            }
-
-        } catch (Throwable e) {
-            String msg = "SavePasswordCommand failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.error(msg, e);
-            return new Answer(cmd, false, msg);
-        }
-        return new Answer(cmd);
-    }
-
-    protected Answer execute(DhcpEntryCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource DhcpEntryCommand: " + _gson.toJson(cmd));
-        }
-
-        // ssh -p 3922 -o StrictHostKeyChecking=no -i $cert root@$domr "/root/edithosts.sh $mac $ip $vm $dfltrt $ns $staticrt" >/dev/null
-        String args = " " + cmd.getVmMac();
-        args += " " + cmd.getVmIpAddress();
-        args += " " + cmd.getVmName();
-        
-        if (cmd.getDefaultRouter() != null) {
-            args += " " + cmd.getDefaultRouter();
-        }
-        
-        if (cmd.getDefaultDns() != null) {
-            args += " " + cmd.getDefaultDns();
-        }
-
-        if (cmd.getStaticRoutes() != null) {
-            args +=  " " + cmd.getStaticRoutes();
-        }
-        
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/edithosts.sh " + args);
-        }
-
-        try {
-            VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-            String controlIp = getRouterSshControlIp(cmd);
-            Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
-                    "/root/edithosts.sh " + args);
-
-            if (!result.first()) {
-                s_logger.error("dhcp_entry command on domR " + controlIp + " failed, message: " + result.second());
-
-                return new Answer(cmd, false, "DhcpEntry failed due to " + result.second());
-            }
-
-            if (s_logger.isInfoEnabled()) {
-                s_logger.info("dhcp_entry command on domain router " + controlIp + " completed");
-            }
-
-        } catch (Throwable e) {
-            String msg = "DhcpEntryCommand failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.error(msg, e);
-            return new Answer(cmd, false, msg);
-        }
-
-        return new Answer(cmd);
-    }
-    
-    protected Answer execute(CheckRouterCommand cmd) {
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Executing resource CheckRouterCommand: " + _gson.toJson(cmd));
-            s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/checkrouter.sh ");
-        }
-
-        Pair<Boolean, String> result;
-        try {
-            VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-            String controlIp = getRouterSshControlIp(cmd);
-            result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
-                    "/root/checkrouter.sh ");
-
-            if (!result.first()) {
-                s_logger.error("check router command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " failed, message: " + result.second());
-
-                return new CheckRouterAnswer(cmd, "CheckRouter failed due to " + result.second());
-            }
-
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("check router command on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " completed");
-            }
-        } catch (Throwable e) {
-            String msg = "CheckRouterCommand failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.error(msg, e);
-            return new CheckRouterAnswer(cmd, msg);
-        }
-        return new CheckRouterAnswer(cmd, result.second(), true);
-    }
-    
-    protected Answer execute(GetDomRVersionCmd cmd) {
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Executing resource GetDomRVersionCmd: " + _gson.toJson(cmd));
-            s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /opt/cloud/bin/get_template_version.sh ");
-        }
-
-        Pair<Boolean, String> result;
-        try {
-            VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-            String controlIp = getRouterSshControlIp(cmd);
-            result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
-                    "/opt/cloud/bin/get_template_version.sh ");
-
-            if (!result.first()) {
-                s_logger.error("GetDomRVersionCmd on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " failed, message: " + result.second());
-
-                return new GetDomRVersionAnswer(cmd, "GetDomRVersionCmd failed due to " + result.second());
-            }
-
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("GetDomRVersionCmd on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " completed");
-            }
-        } catch (Throwable e) {
-            String msg = "GetDomRVersionCmd failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.error(msg, e);
-            return new GetDomRVersionAnswer(cmd, msg);
-        }
-        String[] lines = result.second().split("&");
-        if (lines.length != 2) {
-            return new GetDomRVersionAnswer(cmd, result.second());
-        }
-        return new GetDomRVersionAnswer(cmd, result.second(), lines[0], lines[1]);
-    }
-    
-    protected Answer execute(BumpUpPriorityCommand cmd) {
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Executing resource BumpUpPriorityCommand: " + _gson.toJson(cmd));
-            s_logger.debug("Run command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", /root/bumpup_priority.sh ");
-        }
-
-        Pair<Boolean, String> result;
-        try {
-            VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-            String controlIp = getRouterSshControlIp(cmd);
-            result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
-                    "/root/bumpup_priority.sh ");
-
-            if (!result.first()) {
-                s_logger.error("BumpUpPriority command on domR " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " failed, message: " + result.second());
-
-                return new Answer(cmd, false, "BumpUpPriorityCommand failed due to " + result.second());
-            }
-
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("BumpUpPriorityCommand on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + " completed");
-            }
-        } catch (Throwable e) {
-            String msg = "BumpUpPriorityCommand failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.error(msg, e);
-            return new Answer(cmd, false, msg);
-        }
-        if (result.second() == null || result.second().isEmpty()) {
-            return new Answer(cmd, true, result.second());
-        }
-        return new Answer(cmd, false, result.second());
-    }
-
-    protected Answer execute(VmDataCommand cmd) {
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource VmDataCommand: " + _gson.toJson(cmd));
-        }
-
-        String routerPrivateIpAddress = cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP);
-        String controlIp = getRouterSshControlIp(cmd);
-        
-        String vmIpAddress = cmd.getVmIpAddress();
-        List<String[]> vmData = cmd.getVmData();
-        String[] vmDataArgs = new String[vmData.size() * 2 + 4];
-        vmDataArgs[0] = "routerIP";
-        vmDataArgs[1] = routerPrivateIpAddress;
-        vmDataArgs[2] = "vmIP";
-        vmDataArgs[3] = vmIpAddress;
-        int i = 4;
-        for (String[] vmDataEntry : vmData) {
-            String folder = vmDataEntry[0];
-            String file = vmDataEntry[1];
-            String contents = (vmDataEntry[2] != null) ? vmDataEntry[2] : "none";
-
-            vmDataArgs[i] = folder + "," + file;
-            vmDataArgs[i + 1] = contents;
-            i += 2;
-        }
-
-        String content = encodeDataArgs(vmDataArgs);
-        String tmpFileName = UUID.randomUUID().toString();
-
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Run vm_data command on domain router " + cmd.getAccessDetail(NetworkElementCommand.ROUTER_IP) + ", data: " + content);
-        }
-
-        try {
-            VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-            SshHelper.scpTo(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/tmp", content.getBytes(), tmpFileName, null);
-
-            try {
-                Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null,
-                        "/root/userdata.py " + tmpFileName);
-
-                if (!result.first()) {
-                    s_logger.error("vm_data command on domain router " + controlIp + " failed. messge: " + result.second());
-                    return new Answer(cmd, false, "VmDataCommand failed due to " + result.second());
-                }
-            } finally {
-
-                SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "rm /tmp/" + tmpFileName);
-            }
-
-            if (s_logger.isInfoEnabled()) {
-                s_logger.info("vm_data command on domain router " + controlIp + " completed");
-            }
-
-        } catch (Throwable e) {
-            String msg = "VmDataCommand failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.error(msg, e);
-            return new Answer(cmd, false, msg);
-        }
-        return new Answer(cmd);
-    }
-
-    private String encodeDataArgs(String[] dataArgs) {
-        StringBuilder sb = new StringBuilder();
-
-        for (String arg : dataArgs) {
-            sb.append(arg);
-            sb.append("\n");
-        }
-
-        return sb.toString();
-    }
-
-    protected CheckSshAnswer execute(CheckSshCommand cmd) {
-        String vmName = cmd.getName();
-        String privateIp = cmd.getIp();
-        int cmdPort = cmd.getPort();
-
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Ping command port, " + privateIp + ":" + cmdPort);
-        }
-
-        try {
-            String result = connect(cmd.getName(), privateIp, cmdPort);
-            if (result != null) {
-                s_logger.error("Can not ping System vm " + vmName + "due to:" + result);
-                return new CheckSshAnswer(cmd, "Can not ping System vm " + vmName + "due to:" + result);
-            }
-        } catch (Exception e) {
-            s_logger.error("Can not ping System vm " + vmName + "due to exception");
-            return new CheckSshAnswer(cmd, e);
-        }
-
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Ping command port succeeded for vm " + vmName);
-        }
-
-        if (VirtualMachineName.isValidRouterName(vmName)) {
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("Execute network usage setup command on " + vmName);
-            }
-            networkUsage(privateIp, "create", null);
-        }
-
-        return new CheckSshAnswer(cmd);
-    }
-
-    private VolumeTO[] validateDisks(VolumeTO[] disks) {
-        List<VolumeTO> validatedDisks = new ArrayList<VolumeTO>();
-
-        for (VolumeTO vol : disks) {
-            if (vol.getPoolUuid() != null && !vol.getPoolUuid().isEmpty()) {
-                validatedDisks.add(vol);
-            } else if (vol.getPoolType() == StoragePoolType.ISO && (vol.getPath() != null && !vol.getPath().isEmpty())) {
-                validatedDisks.add(vol);
-            } else {
-                if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Drop invalid disk option, volumeTO: " + _gson.toJson(vol));
-                }
-            }
-        }
-
-        return validatedDisks.toArray(new VolumeTO[0]);
-    }
-
-    protected StartAnswer execute(StartCommand cmd) {
-
-        if (s_logger.isInfoEnabled()) {
-            s_logger.info("Executing resource StartCommand: " + _gson.toJson(cmd));
-        }
-
-        VirtualMachineTO vmSpec = cmd.getVirtualMachine();
-        String vmName = vmSpec.getName();
-        
-        State state = State.Stopped;
-        VmwareContext context = getServiceContext();
-        try {
-            VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-
-            // mark VM as starting state so that sync() can know not to report stopped too early
-            synchronized (_vms) {
-                _vms.put(vmName, State.Starting);
-            }
-
-            VirtualEthernetCardType nicDeviceType = VirtualEthernetCardType.valueOf(vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER));
-            if(s_logger.isDebugEnabled())
-            	s_logger.debug("VM " + vmName + " will be started with NIC device type: " + nicDeviceType);
-            
-            VmwareHypervisorHost hyperHost = getHyperHost(context);
-            VolumeTO[] disks = validateDisks(vmSpec.getDisks());
-            assert (disks.length > 0);
-            NicTO[] nics = vmSpec.getNics();
-
-            HashMap<String, Pair<ManagedObjectReference, DatastoreMO>> dataStoresDetails = inferDatastoreDetailsFromDiskInfo(hyperHost, context, disks);
-            if ((dataStoresDetails == null) || (dataStoresDetails.isEmpty()) ){
-                String msg = "Unable to locate datastore details of the volumes to be attached";
-                s_logger.error(msg);
-                throw new Exception(msg);
-            }
-
-            VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName);
-            if (vmMo != null) {
-                s_logger.info("VM " + vmName + " already exists, tear down devices for reconfiguration");
-                if (getVmState(vmMo) != State.Stopped)
-                    vmMo.safePowerOff(_shutdown_waitMs);
-                vmMo.tearDownDevices(new Class<?>[] { VirtualDisk.class, VirtualEthernetCard.class });
-                vmMo.ensureScsiDeviceController();
-            } else {
-                ManagedObjectReference morDc = hyperHost.getHyperHostDatacenter();
-                assert (morDc != null);
-
-                vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
-                if (vmMo != null) {
-                    if (s_logger.isInfoEnabled()) {
-                        s_logger.info("Found vm " + vmName + " at other host, relocate to " + hyperHost.getHyperHostName());
-                    }
-
-                    takeVmFromOtherHyperHost(hyperHost, vmName);
-
-                    if (getVmState(vmMo) != State.Stopped)
-                        vmMo.safePowerOff(_shutdown_waitMs);
-                    vmMo.tearDownDevices(new Class<?>[] { VirtualDisk.class, VirtualEthernetCard.class });
-                    vmMo.ensureScsiDeviceController();
-                } else {
-                    int ramMb = (int) (vmSpec.getMinRam() / (1024 * 1024));
-                    Pair<ManagedObjectReference, DatastoreMO> rootDiskDataStoreDetails = null;
-                    for (VolumeTO vol : disks) {
-                        if (vol.getType() == Volume.Type.ROOT) {
-                        	rootDiskDataStoreDetails = dataStoresDetails.get(vol.getPoolUuid());
-                        }
-                    }
-
-                    assert (vmSpec.getSpeed() != null) && (rootDiskDataStoreDetails != null);
-                    if (!hyperHost.createBlankVm(vmName, vmSpec.getCpus(), vmSpec.getSpeed().intValue(), 
-                    		getReserveCpuMHz(vmSpec.getSpeed().intValue()), vmSpec.getLimitCpuUse(), ramMb, getReserveMemMB(ramMb),
-                    	translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs()).toString(), rootDiskDataStoreDetails.first(), false)) {
-                        throw new Exception("Failed to create VM. vmName: " + vmName);
-                    }
-                }
-
-                vmMo = hyperHost.findVmOnHyperHost(vmName);
-                if (vmMo == null) {
-                    throw new Exception("Failed to find the newly create or relocated VM. vmName: " + vmName);
-                }
-            }
-
-            int totalChangeDevices = disks.length + nics.length;
-            VolumeTO volIso = null;
-            if (vmSpec.getType() != VirtualMachine.Type.User) {
-                // system VM needs a patch ISO
-                totalChangeDevices++;
-            } else {
-                for (VolumeTO vol : disks) {
-                    if (vol.getType() == Volume.Type.ISO) {
-                        volIso = vol;
-                        break;
-                    }
-                }
-
-                if (volIso == null)
-                    totalChangeDevices++;
-            }
-
-            VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();
-            int ramMb = (int) (vmSpec.getMinRam() / (1024 * 1024));
-            VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getSpeed().intValue(), 
-            	getReserveCpuMHz(vmSpec.getSpeed().intValue()), ramMb, getReserveMemMB(ramMb),
-        		translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs()).toString(), vmSpec.getLimitCpuUse());
-            
-            VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices];
-            int i = 0;
-            int ideControllerKey = vmMo.getIDEDeviceControllerKey();
-            int scsiControllerKey = vmMo.getScsiDeviceControllerKey();
-            int controllerKey;
-            String datastoreDiskPath;
-
-            // prepare systemvm patch ISO
-            if (vmSpec.getType() != VirtualMachine.Type.User) {
-                // attach ISO (for patching of system VM)
-                String secStoreUrl = mgr.getSecondaryStorageStoreUrl(Long.parseLong(_dcId));
-                if(secStoreUrl == null) {
-                    String msg = "secondary storage for dc " + _dcId + " is not ready yet?";
-                    throw new Exception(msg);
-                }
-                mgr.prepareSecondaryStorageStore(secStoreUrl);
-                
-                ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl);
-                if (morSecDs == null) {
-                    String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl;
-                    throw new Exception(msg);
-                }
-                DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs);
-
-                deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
-                Pair<VirtualDevice, Boolean> isoInfo = VmwareHelper.prepareIsoDevice(vmMo, String.format("[%s] systemvm/%s", secDsMo.getName(), mgr.getSystemVMIsoFileNameOnDatastore()), 
-                	secDsMo.getMor(), true, true, i, i + 1);
-                deviceConfigSpecArray[i].setDevice(isoInfo.first());
-                if (isoInfo.second()) {
-                	if(s_logger.isDebugEnabled())
-                		s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first()));
-                    deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
-                } else {
-                	if(s_logger.isDebugEnabled())
-                		s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
-                    deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.edit);
-                }
-                i++;
-            } else {
-                // we will always plugin a CDROM device
-                if (volIso != null && volIso.getPath() != null && !volIso.getPath().isEmpty()) {
-                    Pair<String, ManagedObjectReference> isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, volIso.getPath());
-                    assert (isoDatastoreInfo != null);
-                    assert (isoDatastoreInfo.second() != null);
-
-                    deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
-                    Pair<VirtualDevice, Boolean> isoInfo = VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, i, i + 1);
-                    deviceConfigSpecArray[i].setDevice(isoInfo.first());
-                    if (isoInfo.second()) {
-                    	if(s_logger.isDebugEnabled())
-                    		s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first()));
-                    	deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
-                    } else {
-                    	if(s_logger.isDebugEnabled())
-                    		s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
-                        deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.edit);
-                    }
-                } else {
-                    deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
-                    Pair<VirtualDevice, Boolean> isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, i, i + 1);
-                    deviceConfigSpecArray[i].setDevice(isoInfo.first());
-                    if (isoInfo.second()) {
-                    	if(s_logger.isDebugEnabled())
-                    		s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
-                    	
-                        deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
-                    } else {
-                    	if(s_logger.isDebugEnabled())
-                    		s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first()));
-                    	
-                        deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.edit);
-                    }
-                }
-                i++;
-            }
-
-            for (VolumeTO vol : sortVolumesByDeviceId(disks)) {
-                deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
-
-                if (vol.getType() == Volume.Type.ISO) {
-                    controllerKey = ideControllerKey;
-                } else {
-                    if(vol.getType() == Volume.Type.ROOT) {
-                    	if(vmSpec.getDetails() != null && vmSpec.getDetails().get(VmDetailConstants.ROOK_DISK_CONTROLLER) != null)
-                    	{
-                    		if(vmSpec.getDetails().get(VmDetailConstants.ROOK_DISK_CONTROLLER).equalsIgnoreCase("scsi"))
-    	                		controllerKey = scsiControllerKey;
-    	                	else
-    	                		controllerKey = ideControllerKey;
-                    	} else {
-                    		controllerKey = scsiControllerKey;
-                    	}
-                    } else {
-                        // DATA volume always use SCSI device
-                        controllerKey = scsiControllerKey;
-                    }
-                }
-
-                if (vol.getType() != Volume.Type.ISO) {
-                    Pair<ManagedObjectReference, DatastoreMO> volumeDsDetails = dataStoresDetails.get(vol.getPoolUuid());
-                    assert (volumeDsDetails != null);
-                	VirtualDevice device;
-                    datastoreDiskPath = String.format("[%s] %s.vmdk", volumeDsDetails.second().getName(), vol.getPath());
-                    String chainInfo = vol.getChainInfo();
-
-                    if (chainInfo != null && !chainInfo.isEmpty()) {
-                        String[] diskChain = _gson.fromJson(chainInfo, String[].class);
-                        if (diskChain == null || diskChain.length < 1) {
-                            s_logger.warn("Empty previously-saved chain info, fall back to the original");
-                            device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), i, i + 1);
-                        } else {
-                            s_logger.info("Attach the disk with stored chain info: " + chainInfo);
-                            for (int j = 0; j < diskChain.length; j++) {
-                                diskChain[j] = String.format("[%s] %s", volumeDsDetails.second().getName(), diskChain[j]);
-                            }
-
-                            device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, diskChain, volumeDsDetails.first(), i, i + 1);
-                        }
-                    } else {
-                        device = VmwareHelper.prepareDiskDevice(vmMo, controllerKey, new String[] { datastoreDiskPath }, volumeDsDetails.first(), i, i + 1);
-                    }
-                    deviceConfigSpecArray[i].setDevice(device);
-                    deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
-
-                	if(s_logger.isDebugEnabled())
-                		s_logger.debug("Prepare volume at new device " + _gson.toJson(device));
-                    
-                    i++;
-                }
-            }
-
-            VirtualDevice nic;
-            int nicMask = 0;
-            int nicCount = 0;
-            for (NicTO nicTo : sortNicsByDeviceId(nics)) {
-                s_logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo));
-
-                Pair<ManagedObjectReference, String> networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo);
-                if (mgr.getNexusVSwitchGlobalParameter()) {
-                    String dvSwitchUuid;
-                    ManagedObjectReference dcMor = hyperHost.getHyperHostDatacenter();
-                    DatacenterMO dataCenterMo = new DatacenterMO(context, dcMor);
-                    ManagedObjectReference dvsMor = dataCenterMo.getDvSwitchMor(networkInfo.first());
-                    dvSwitchUuid = dataCenterMo.getDvSwitchUuid(dvsMor);
-                    s_logger.info("Preparing NIC device on dvSwitch : " + dvSwitchUuid);
-                    nic = VmwareHelper.prepareDvNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), dvSwitchUuid, nicTo.getMac(), i, i + 1, true, true);
-                } else {
-                    s_logger.info("Preparing NIC device on network " + networkInfo.second());
-                    nic = VmwareHelper.prepareNicDevice(vmMo, networkInfo.first(), nicDeviceType, networkInfo.second(), nicTo.getMac(), i, i + 1, true, true);
-                }
-                
-                deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec();
-                deviceConfigSpecArray[i].setDevice(nic);
-                deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.add);
-                
-            	if(s_logger.isDebugEnabled())
-            		s_logger.debug("Prepare NIC at new device " + _gson.toJson(deviceConfigSpecArray[i]));
- 
-            	// this is really a hacking for DomR, upon DomR startup, we will reset all the NIC allocation after eth3
-                if(nicCount < 3)
-                	nicMask |= (1 << nicCount);
-                
-                i++;
-                nicCount++;
-            }
-
-            vmConfigSpec.setDeviceChange(deviceConfigSpecArray);
-
-            // pass boot arguments through machine.id & perform customized options to VMX
-    
-            Map<String, String> vmDetailOptions = validateVmDetails(vmSpec.getDetails());
-            OptionValue[] extraOptions = new OptionValue[2 + vmDetailOptions.size()];
-            extraOptions[0] = new OptionValue();
-            extraOptions[0].setKey("machine.id");
-            extraOptions[0].setValue(vmSpec.getBootArgs());
-            
-            extraOptions[1] = new OptionValue();
-            extraOptions[1].setKey("devices.hotplug");
-            extraOptions[1].setValue("true");
-
-            int j = 2;
-            for(Map.Entry<String, String> entry : vmDetailOptions.entrySet()) {
-            	extraOptions[j] = new OptionValue();
-            	extraOptions[j].setKey(entry.getKey());
-            	extraOptions[j].setValue(entry.getValue());
-            	j++;
-            }
-            
-            String keyboardLayout = null;
-            if(vmSpec.getDetails() != null)
-            	keyboardLayout = vmSpec.getDetails().get(VmDetailConstants.KEYBOARD);
-            vmConfigSpec.setExtraConfig(configureVnc(extraOptions, hyperHost, vmName, vmSpec.getVncPassword(), keyboardLayout));
-
-            if (!vmMo.configureVm(vmConfigSpec)) {
-                throw new Exception("Failed to configure VM before start. vmName: " + vmName);
-            }
-            
-            vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, String.valueOf(nicMask));
-
-            if (!vmMo.powerOn()) {
-                throw new Exception("Failed to start VM. vmName: " + vmName);
-            }
-
-            state = State.Running;
-            return new StartAnswer(cmd);
-        } catch (Throwable e) {
-            if (e instanceof RemoteException) {
-                s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
-                invalidateServiceContext();
-            }
-
-            String msg = "StartCommand failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.warn(msg, e);
-            return new StartAnswer(cmd, msg);
-        } finally {
-            synchronized (_vms) {
-                if (state != State.Stopped) {
-                    _vms.put(vmName, state);
-                } else {
-                    _vms.remove(vmName);
-                }
-            }
-        }
-    }
-    
-    private Map<String, String> validateVmDetails(Map<String, String> vmDetails) {
-    	Map<String, String> validatedDetails = new HashMap<String, String>();
-    	
-    	if(vmDetails != null && vmDetails.size() > 0) {
-    		for(Map.Entry<String, String> entry : vmDetails.entrySet()) {
-    			if("machine.id".equalsIgnoreCase(entry.getKey()))
-    				continue;
-    			else if("devices.hotplug".equalsIgnoreCase(entry.getKey()))
-    				continue;
-    			else if("RemoteDisplay.vnc.enabled".equalsIgnoreCase(entry.getKey()))
-    				continue;
-    			else if("RemoteDisplay.vnc.password".equalsIgnoreCase(entry.getKey()))
-    				continue;
-    			else if("RemoteDisplay.vnc.port".equalsIgnoreCase(entry.getKey()))
-    				continue;
-    			else if("RemoteDisplay.vnc.keymap".equalsIgnoreCase(entry.getKey()))
-    				continue;
-    			else
-    				validatedDetails.put(entry.getKey(), entry.getValue());
-    		}
-    	}
-    	return validatedDetails;
-    }
-
-    private int getReserveCpuMHz(int cpuMHz) {
-    	if(this._reserveCpu) {
-    		return (int)(cpuMHz / this._cpuOverprovisioningFactor);
-    	}
-    	
-    	return 0;
-    }
-    
-    private int getReserveMemMB(int memMB) {
-    	if(this._reserveMem) {
-    		return (int)(memMB / this._memOverprovisioningFactor);
-    	}
-    	
-    	return 0;
-    }
-    
-    private NicTO[] sortNicsByDeviceId(NicTO[] nics) {
-
-        List<NicTO> listForSort = new ArrayList<NicTO>();
-        for (NicTO nic : nics) {
-            listForSort.add(nic);
-        }
-        Collections.sort(listForSort, new Comparator<NicTO>() {
-
-            @Override
-            public int compare(NicTO arg0, NicTO arg1) {
-                if (arg0.getDeviceId() < arg1.getDeviceId()) {
-                    return -1;
-                } else if (arg0.getDeviceId() == arg1.getDeviceId()) {
-                    return 0;
-                }
-
-                return 1;
-            }
-        });
-
-        return listForSort.toArray(new NicTO[0]);
-    }
-    
-    private VolumeTO[] sortVolumesByDeviceId(VolumeTO[] volumes) {
-
-        List<VolumeTO> listForSort = new ArrayList<VolumeTO>();
-        for (VolumeTO vol : volumes) {
-            listForSort.add(vol);
-        }
-        Collections.sort(listForSort, new Comparator<VolumeTO>() {
-
-            @Override
-            public int compare(VolumeTO arg0, VolumeTO arg1) {
-                if (arg0.getDeviceId() < arg1.getDeviceId()) {
-                    return -1;
-                } else if (arg0.getDeviceId() == arg1.getDeviceId()) {
-                    return 0;
-                }
-
-                return 1;
-            }
-        });
-
-        return listForSort.toArray(new VolumeTO[0]);
-    }
-
-    private HashMap<String, Pair<ManagedObjectReference, DatastoreMO>> inferDatastoreDetailsFromDiskInfo(VmwareHypervisorHost hyperHost, VmwareContext context, VolumeTO[] disks) throws Exception {
-        HashMap<String ,Pair<ManagedObjectReference, DatastoreMO>> poolMors = new HashMap<String, Pair<ManagedObjectReference, DatastoreMO>>();
-
-        assert (hyperHost != null) && (context != null);
-        for (VolumeTO vol : disks) {
-            if (vol.getType() != Volume.Type.ISO) {
-                String poolUuid = vol.getPoolUuid();
-                if(poolMors.get(poolUuid) == null) {
-                    ManagedObjectReference morDataStore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolUuid);
-                    if (morDataStore == null) {
-                        String msg = "Failed to get the mounted datastore for the volume's pool " + poolUuid;
-                        s_logger.error(msg);
-                        throw new Exception(msg);
-                    }
-                    poolMors.put(vol.getPoolUuid(), new Pair<ManagedObjectReference, DatastoreMO> (morDataStore, new DatastoreMO(context, morDataStore)));
-                }
-            }
-        }
-        return poolMors;
-    }
-
-    private String getVlanInfo(NicTO nicTo, String defaultVlan) {
-        if (nicTo.getBroadcastType() == BroadcastDomainType.Native) {
-            return defaultVlan;
-        }
-
-        if (nicTo.getBroadcastType() == BroadcastDomainType.Vlan) {
-            if (nicTo.getBroadcastUri() != null) {
-                return nicTo.getBroadcastUri().getHost();
-            } else {
-                s_logger.warn("BroadcastType is not claimed as VLAN, but without vlan info in broadcast URI. Use vlan info from labeling: " + defaultVlan);
-                return defaultVlan;
-            }
-        }
-
-        s_logger.warn("Unrecognized broadcast type in VmwareResource, type: " + nicTo.getBroadcastType().toString() + ". Use vlan info from labeling: " + defaultVlan);
-        return defaultVlan;
-    }
-
-    private Pair<ManagedObjectReference, String> prepareNetworkFromNicInfo(HostMO hostMo, NicTO nicTo) throws Exception {
-        
-        Pair<String, String> switchName =  getTargetSwitch(nicTo);
-        String namePrefix = getNetworkNamePrefix(nicTo);
-        Pair<ManagedObjectReference, String> networkInfo = null;
-
-        s_logger.info("Prepare network on vSwitch: " + switchName + " with name prefix: " + namePrefix);
-        
-        if(!_nexusVSwitch) {
-        	networkInfo = HypervisorHostHelper.prepareNetwork(switchName.first(), namePrefix, hostMo, getVlanInfo(nicTo, switchName.second()), 
-                    nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), _ops_timeout, 
-                    !namePrefix.startsWith("cloud.private"));
-        }
-        else {
-        	networkInfo = HypervisorHostHelper.prepareNetwork(switchName.first(), namePrefix, hostMo, getVlanInfo(nicTo, switchName.second()), 
-                    nicTo.getNetworkRateMbps(), nicTo.getNetworkRateMulticastMbps(), _ops_timeout);
-        }
-        	
-        return networkInfo;
-    }
-    
-    // return Pair<switch name, vlan tagging>
-    private Pair<String, String> getTargetSwitch(NicTO nicTo) throws Exception {
-        if(nicTo.getName() != null && !nicTo.getName().isEmpty()) {
-        	String[] tokens = nicTo.getName().split(",");
-        	
-        	if(tokens.length == 2) {
-        		return new Pair<String, String>(tokens[0], tokens[1]);
-        	} else {
-        		return new Pair<String, String>(nicTo.getName(), Vlan.UNTAGGED);
-        	}
-        }
-        
-        if (nicTo.getType() == Networks.TrafficType.Guest) {
-            return new Pair<String, String>(this._guestNetworkVSwitchName, Vlan.UNTAGGED);
-        } else if (nicTo.getType() == Networks.TrafficType.Control || nicTo.getType() == Networks.TrafficType.Management) {
-            return new Pair<String, String>(this._privateNetworkVSwitchName, Vlan.UNTAGGED);
-        } else if (nicTo.getType() == Networks.TrafficType.Public) {
-            return new Pair<String, String>(this._publicNetworkVSwitchName, Vlan.UNTAGGED);
-        } else if (nicTo.getType() == Networks.TrafficType.Storage) {
-            return new Pair<String, String>(this._privateNetworkVSwitchName, Vlan.UNTAGGED);
-        } else if (nicTo.getType() == Networks.TrafficType.Vpn) {
-            throw new Exception("Unsupported traffic type: " + nicTo.getType().toString());
-        } else {
-            throw new Exception("Unsupported traffic type: " + nicTo.getType().toString());
-        }
-    }
-    
-    private String getNetworkNamePrefix(NicTO nicTo) throws Exception {
-        if (nicTo.getType() == Networks.TrafficType.Guest) {
-            return "cloud.guest";
-        } else if (nicTo.getType() == Networks.TrafficType.Control || nicTo.getType() == Networks.TrafficType.Management) {
-            return "cloud.private";
-        } else if (nicTo.getType() == Networks.TrafficType.Public) {
-            return "cloud.public";
-        } else if (nicTo.getType() == Networks.TrafficType.Storage) {
-            return "cloud.storage";
-        } else if (nicTo.getType() == Networks.TrafficType.Vpn) {
-            throw new Exception("Unsupported traffic type: " + nicTo.getType().toString());
-        } else {
-            throw new Exception("Unsupported traffic type: " + nicTo.getType().toString());
-        }
-    }
-
-    protected synchronized Answer execute(final RemoteAccessVpnCfgCommand cmd) {
-        String controlIp = getRouterSshControlIp(cmd);
-        StringBuffer argsBuf = new StringBuffer();
-        if (cmd.isCreate()) {
-            argsBuf.append(" -r ").append(cmd.getIpRange()).append(" -p ").append(cmd.getPresharedKey()).append(" -s ").append(cmd.getVpnServerIp()).append(" -l ").append(cmd.getLocalIp())
-            .append(" -c ");
-
-        } else {
-            argsBuf.append(" -d ").append(" -s ").append(cmd.getVpnServerIp());
-        }
-
-        try {
-            VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-
-            if (s_logger.isDebugEnabled()) {
-                s_logger.debug("Executing /opt/cloud/bin/vpn_lt2p.sh ");
-            }
-
-            Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/opt/cloud/bin/vpn_l2tp.sh " + argsBuf.toString());
-
-            if (!result.first()) {
-                s_logger.error("RemoteAccessVpnCfg command on domR failed, message: " + result.second());
-
-                return new Answer(cmd, false, "RemoteAccessVpnCfg command failed due to " + result.second());
-            }
-
-            if (s_logger.isInfoEnabled()) {
-                s_logger.info("RemoteAccessVpnCfg command on domain router " + argsBuf.toString() + " completed");
-            }
-
-        } catch (Throwable e) {
-            if (e instanceof RemoteException) {
-                s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
-                invalidateServiceContext();
-            }
-
-            String msg = "RemoteAccessVpnCfg command failed due to " + VmwareHelper.getExceptionMessage(e);
-            s_logger.error(msg, e);
-            return new Answer(cmd, false, msg);
-        }
-
-        return new Answer(cmd);
-    }
-
-    protected synchronized Answer execute(final VpnUsersCfgCommand cmd) {
-        VmwareManager mgr = getServiceContext().getStockObject(VmwareManager.CONTEXT_STOCK_NAME);
-
-        String controlIp = getRouterSshControlIp(cmd);
-        for (VpnUsersCfgCommand.UsernamePassword userpwd : cmd.getUserpwds()) {
-            StringBuffer argsBuf = new StringBuffer();
-            if (!userpwd.isAdd()) {
-                argsBuf.append(" -U ").append(userpwd.getUsername());
-            } else {
-                argsBuf.append(" -u ").append(userpwd.getUsernamePassword());
-            }
-
-            try {
-
-                if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Executing /opt/cloud/bin/vpn_lt2p.sh ");
-                }
-
-                Pair<Boolean, String> result = SshHelper.sshExecute(controlIp, DEFAULT_DOMR_SSHPORT, "root", mgr.getSystemVMKeyFile(), null, "/opt/cloud/bin/vpn_l2tp.sh " + argsBuf.toString());
-
-                if (!result.first()) {
-                    s_logger.error("VpnUserCfg command on domR failed, message: " + result.second());
-
-                    return new Answer(cmd, false, "VpnUserCfg command failed due to " + result.second());
-                }
-            } catch (Throwable e) {
-                if (e instanceof RemoteException) {
-                    s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context");
-                    invalidateServiceContext();
-                }
-
-                String msg = "VpnUserCfg command failed due to " + VmwareHelper.getExceptionMessage(e);
-                s_logger.error(msg, e);
-                return new Answer(cmd, false, msg);
-            }
-        }
-
-        return new Answer(cmd);
-    }
-
-    private VirtualMachineMO takeVmFromOtherHyperHost(VmwareHypervisorHost hyperHost, String vmName) throws Exception {
-
-        VirtualMachineMO vmMo = hyperHost.findVmOnPeerHyperHost(vmName);
-        if (vmMo != null) {
-            ManagedObjectReference morTargetPhysicalHost = hyperHost.findMigrationTarget(vmMo);
-            if (morTargetPhysicalHost == null) {
-                String msg = "VM " + vmName + " is on other host and we have no resource available to migrate and start it here";
-                s_logger.error(msg);
-                throw new Exception(msg);
-            }
-
-            if (!vmMo.relocate(morTargetPhysicalHost)) {
-                String msg = "VM " + vmName + " is on other host and we failed to relocate it here";
-                s_logger.error(msg);
-                throw new Exception(msg);
-            }
-
-            return vmMo;
-        }
-        return null;
-    }
-
-    // isoUrl sample content :
-    // nfs://192.168.10.231/export/home/kelven/vmware-test/secondary/template/tmpl/2/200//200-2-80f7ee58-6eff-3a2d-bcb0-59663edf6d26.iso
-    private Pair<String, ManagedObjectReference> getIsoDatastoreInfo(VmwareHypervisorHost hyperHost, String isoUrl) throws Exception {
-
-        assert (isoUrl != null);
-        int isoFileNameStartPos = isoUrl.lastIndexOf("/");
-        if (isoFileNameStartPos < 0) {
-            throw new Exception("Invalid ISO path info");
-        }
-
-        String isoFileName = isoUrl.substring(isoFileNameStartPos);
-
-        int templateRootPos = isoUrl.indexOf("template/tmpl");
-        if (templateRootPos < 0) {
-            throw new Exception("Invalid ISO path info");
-        }
-
-        String storeUrl = isoUrl.substring(0, templateRootPos - 1);
-        String isoPath = isoUrl.substring(templateRootPos,

<TRUNCATED>