You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ke...@apache.org on 2014/03/14 01:14:46 UTC
[01/10] git commit: updated refs/heads/master to 42d6373
Repository: cloudstack
Updated Branches:
refs/heads/master 8ff94601f -> 42d637376
Handle a race-condition in VirtualNetworkManagerImpl.java in the process of starting it parallelly.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/42d63737
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/42d63737
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/42d63737
Branch: refs/heads/master
Commit: 42d637376bba7a975830d695f14be7678dc6e110
Parents: 5a75a3e
Author: Kelven Yang <ke...@gmail.com>
Authored: Wed Mar 12 16:09:51 2014 -0700
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
.../VirtualNetworkApplianceManagerImpl.java | 117 ++++++++++++-------
1 file changed, 76 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/42d63737/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
index 74cfd74..eeab91d 100755
--- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
+++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java
@@ -17,6 +17,44 @@
package com.cloud.network.router;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import javax.ejb.Local;
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+import org.apache.log4j.Logger;
+import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd;
+import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
+import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
+import org.apache.cloudstack.framework.config.ConfigDepot;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.framework.jobs.AsyncJobManager;
+import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.utils.identity.ManagementServerNode;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
import com.cloud.agent.api.AgentControlAnswer;
@@ -238,45 +276,7 @@ import com.cloud.vm.dao.NicIpAliasVO;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.UserVmDetailsDao;
import com.cloud.vm.dao.VMInstanceDao;
-import org.apache.cloudstack.api.command.admin.router.RebootRouterCmd;
-import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
-import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
import org.apache.cloudstack.config.ApiServiceConfiguration;
-import org.apache.cloudstack.context.CallContext;
-import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
-import org.apache.cloudstack.framework.config.ConfigDepot;
-import org.apache.cloudstack.framework.config.ConfigKey;
-import org.apache.cloudstack.framework.config.Configurable;
-import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
-import org.apache.cloudstack.framework.jobs.AsyncJobManager;
-import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO;
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.utils.identity.ManagementServerNode;
-import org.apache.log4j.Logger;
-
-import javax.ejb.Local;
-import javax.inject.Inject;
-import javax.naming.ConfigurationException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
/**
* VirtualNetworkApplianceManagerImpl manages the different types of virtual network appliances available in the Cloud Stack.
@@ -1885,8 +1885,35 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
return new Pair<DeploymentPlan, List<DomainRouterVO>>(plan, routers);
}
- private DomainRouterVO startVirtualRouter(final DomainRouterVO router, final User user, final Account caller, final Map<Param, Object> params) throws StorageUnavailableException,
- InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {
+ private DomainRouterVO waitRouter(DomainRouterVO router) {
+ DomainRouterVO vm = _routerDao.findById(router.getId());
+
+ if (s_logger.isDebugEnabled())
+ s_logger.debug("Router " + router.getInstanceName() + " is not fully up yet, we will wait");
+ while (vm.getState() == State.Starting) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+
+ // reload to get the latest state info
+ vm = _routerDao.findById(router.getId());
+ }
+
+ if (vm.getState() == State.Running) {
+ if (s_logger.isDebugEnabled())
+ s_logger.debug("Router " + router.getInstanceName() + " is now fully up");
+
+ return router;
+ }
+
+ s_logger.warn("Router " + router.getInstanceName() + " failed to start. current state: " + vm.getState());
+ return null;
+ }
+
+ private DomainRouterVO startVirtualRouter(DomainRouterVO router, User user, Account caller, Map<Param, Object> params)
+ throws StorageUnavailableException, InsufficientCapacityException,
+ ConcurrentOperationException, ResourceUnavailableException {
if (router.getRole() != Role.VIRTUAL_ROUTER || !router.getIsRedundantRouter()) {
return this.start(router, user, caller, params, null);
@@ -1897,7 +1924,15 @@ public class VirtualNetworkApplianceManagerImpl extends ManagerBase implements V
return router;
}
- final DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null);
+ //
+ // If another thread has already requested a VR start, there is a transition period for VR to transit from
+ // Starting to Running, there exist a race conditioning window here
+ // We will wait until VR is up or fail
+ if (router.getState() == State.Starting) {
+ return waitRouter(router);
+ }
+
+ DataCenterDeployment plan = new DataCenterDeployment(0, null, null, null, null, null);
DomainRouterVO result = null;
assert router.getIsRedundantRouter();
final List<Long> networkIds = _routerDao.getRouterNetworks(router.getId());
[09/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
Give graceful state transition period to live with race-condition on VM startup time
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/5a75a3e1
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/5a75a3e1
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/5a75a3e1
Branch: refs/heads/master
Commit: 5a75a3e1f92dc8f6352f1e27797c44d40565454b
Parents: 3123c30
Author: Kelven Yang <ke...@gmail.com>
Authored: Wed Mar 12 14:30:18 2014 -0700
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
.../vm/VirtualMachinePowerStateSyncImpl.java | 44 +++++++++++++++++---
.../vmware/resource/VmwareResource.java | 2 -
2 files changed, 38 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5a75a3e1/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
----------------------------------------------------------------------
diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
index 6332958..9edecac 100644
--- a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
+++ b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java
@@ -16,6 +16,7 @@
// under the License.
package com.cloud.vm;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -24,10 +25,12 @@ import javax.inject.Inject;
import org.apache.log4j.Logger;
+import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.messagebus.MessageBus;
import org.apache.cloudstack.framework.messagebus.PublishScope;
import com.cloud.agent.api.HostVmStateReportEntry;
+import com.cloud.utils.DateUtil;
import com.cloud.vm.dao.VMInstanceDao;
public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStateSync {
@@ -37,6 +40,9 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
@Inject VMInstanceDao _instanceDao;
@Inject VirtualMachineManager _vmMgr;
+ protected final ConfigKey<Integer> PingInterval = new ConfigKey<Integer>(Integer.class, "ping.interval", "Advanced", "60",
+ "Interval to send application level pings to make sure the connection is still working", false);
+
public VirtualMachinePowerStateSyncImpl() {
}
@@ -96,15 +102,41 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
}
if (vmsThatAreMissingReport.size() > 0) {
+ Date currentTime = DateUtil.currentGMTTime();
+ if (s_logger.isDebugEnabled())
+ s_logger.debug("Run missing VM report. current time: " + currentTime.getTime());
+
+ // 2 times of sync-update interval for graceful period
+ long milliSecondsGracefullPeriod = PingInterval.value() * 2000;
+
for (VMInstanceVO instance : vmsThatAreMissingReport) {
- if (_instanceDao.updatePowerState(instance.getId(), hostId, VirtualMachine.PowerState.PowerReportMissing)) {
- if (s_logger.isDebugEnabled())
- s_logger.debug("VM state report is updated. host: " + hostId + ", vm id: " + instance.getId() + ", power state: PowerReportMissing ");
- _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, instance.getId());
+ Date vmStateUpdateTime = instance.getUpdateTime();
+ if (vmStateUpdateTime == null) {
+ s_logger.warn("VM state was updated but update time is null?! vm id: " + instance.getId());
+ vmStateUpdateTime = currentTime;
+ }
+
+ if (s_logger.isDebugEnabled())
+ s_logger.debug("Detected missing VM. host: " + hostId + ", vm id: " + instance.getId() +
+ ", power state: PowerReportMissing, last state update: " + vmStateUpdateTime.getTime());
+
+ long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime();
+
+ if (milliSecondsSinceLastStateUpdate > milliSecondsGracefullPeriod) {
+ s_logger.debug("vm id: " + instance.getId() + " - time since last state update(" + milliSecondsSinceLastStateUpdate + "ms) has passed graceful period");
+
+ if (_instanceDao.updatePowerState(instance.getId(), hostId, VirtualMachine.PowerState.PowerReportMissing)) {
+ if (s_logger.isDebugEnabled())
+ s_logger.debug("VM state report is updated. host: " + hostId + ", vm id: " + instance.getId() + ", power state: PowerReportMissing ");
+
+ _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, instance.getId());
+ } else {
+ if (s_logger.isDebugEnabled())
+ s_logger.debug("VM power state does not change, skip DB writing. vm id: " + instance.getId());
+ }
} else {
- if (s_logger.isDebugEnabled())
- s_logger.debug("VM power state does not change, skip DB writing. vm id: " + instance.getId());
+ s_logger.debug("vm id: " + instance.getId() + " - time since last state update(" + milliSecondsSinceLastStateUpdate + "ms) has not passed graceful period yet");
}
}
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5a75a3e1/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
index 4693f86..0a2cad4 100755
--- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
+++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java
@@ -41,7 +41,6 @@ import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-
import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
@@ -2681,7 +2680,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa
try {
VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName());
if (vmMo != null) {
-
State state = null;
synchronized (_vms) {
state = _vms.get(cmd.getVmName());
[05/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
move HA rescheduling operation to HA worker context.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3d5fbe70
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3d5fbe70
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3d5fbe70
Branch: refs/heads/master
Commit: 3d5fbe7073d4b7234611e972ca030088bff2f3f2
Parents: 53f306e
Author: Kelven Yang <ke...@gmail.com>
Authored: Tue Mar 4 15:58:03 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
.../cloud/ha/HighAvailabilityManagerImpl.java | 22 +++++++++++++++++---
1 file changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3d5fbe70/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
index b7d5be8..3bcaf2e 100755
--- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -267,10 +267,17 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
"] is down." +
((sb != null) ? sb.toString() : ""));
- for (final VMInstanceVO vm : vms) {
+ for (VMInstanceVO vm : vms) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Notifying HA Mgr of to restart vm " + vm.getId() + "-" + vm.getHostName());
}
+ vm = _instanceDao.findByUuid(vm.getUuid());
+ Long hostId = vm.getHostId();
+ if ( hostId != null && !hostId.equals(host.getId()) ) {
+ s_logger.debug("VM " + vm.getHostName() + " is not on down host " + host.getId() + " it is on other host "
+ + hostId + " VM HA is done");
+ continue;
+ }
scheduleRestart(vm, investigate);
}
}
@@ -951,10 +958,19 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
@Override
public boolean postStateTransitionEvent(State oldState, VirtualMachine.Event event, State newState, VirtualMachine vo, boolean status, Object opaque) {
if (oldState == State.Running && event == VirtualMachine.Event.FollowAgentPowerOffReport && newState == State.Stopped) {
- VMInstanceVO vm = _instanceDao.findById(vo.getId());
+ final VMInstanceVO vm = _instanceDao.findById(vo.getId());
if (vm.isHaEnabled()) {
s_logger.info("Detected out-of-band stop of a HA enabled VM " + vm.getInstanceName() + ", will schedule restart");
- scheduleRestart(vm, true);
+ _executor.submit(new ManagedContextRunnable() {
+ @Override
+ protected void runInContext() {
+ try {
+ scheduleRestart(vm, true);
+ } catch (Exception e) {
+ s_logger.warn("Unexpected exception when scheduling a HA restart", e);
+ }
+ }
+ });
}
}
return true;
[06/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
Seperate job executor pools to avoid thread starvation situation.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/8e27120b
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/8e27120b
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/8e27120b
Branch: refs/heads/master
Commit: 8e27120be40725c7e89db8a69b6a976cade8e917
Parents: 6ad245e
Author: Kelven Yang <ke...@gmail.com>
Authored: Thu Mar 6 10:36:39 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
.../framework/jobs/AsyncJobManager.java | 3 ++-
.../jobs/impl/AsyncJobManagerImpl.java | 21 ++++++++++++++------
.../framework/jobs/impl/AsyncJobMonitor.java | 2 +-
3 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8e27120b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java
index 67733ed..fe5c067 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/AsyncJobManager.java
@@ -27,7 +27,8 @@ import com.cloud.utils.component.Manager;
public interface AsyncJobManager extends Manager {
- public static final String JOB_POOL_THREAD_PREFIX = "Job-Executor";
+ public static final String API_JOB_POOL_THREAD_PREFIX = "API-Job-Executor";
+ public static final String WORK_JOB_POOL_THREAD_PREFIX = "Work-Job-Executor";
AsyncJobVO getAsyncJob(long jobId);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8e27120b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
index b9246aa..9b9460c 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
@@ -117,7 +117,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
private volatile long _executionRunNumber = 1;
private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AsyncJobMgr-Heartbeat"));
- private ExecutorService _executor;
+ private ExecutorService _apiJobExecutor;
+ private ExecutorService _workerJobExecutor;
@Override
public String getConfigComponentName() {
@@ -390,7 +391,10 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
if (executeInContext) {
runnable.run();
} else {
- _executor.submit(runnable);
+ if (job.getDispatcher() == null || job.getDispatcher().equalsIgnoreCase("ApiAsyncJobDispatcher"))
+ _apiJobExecutor.submit(runnable);
+ else
+ _workerJobExecutor.submit(runnable);
}
}
@@ -855,10 +859,14 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
final Properties dbProps = DbProperties.getDbProperties();
final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive"));
- int poolSize = (cloudMaxActive * 2) / 3;
+ int apiPoolSize = cloudMaxActive / 2;
+ int workPoolSize = (cloudMaxActive * 2) / 3;
- s_logger.info("Start AsyncJobManager thread pool in size " + poolSize);
- _executor = Executors.newFixedThreadPool(poolSize, new NamedThreadFactory(AsyncJobManager.JOB_POOL_THREAD_PREFIX));
+ s_logger.info("Start AsyncJobManager API executor thread pool in size " + apiPoolSize);
+ _apiJobExecutor = Executors.newFixedThreadPool(apiPoolSize, new NamedThreadFactory(AsyncJobManager.API_JOB_POOL_THREAD_PREFIX));
+
+ s_logger.info("Start AsyncJobManager Work executor thread pool in size " + workPoolSize);
+ _workerJobExecutor = Executors.newFixedThreadPool(workPoolSize, new NamedThreadFactory(AsyncJobManager.WORK_JOB_POOL_THREAD_PREFIX));
} catch (final Exception e) {
throw new ConfigurationException("Unable to load db.properties to configure AsyncJobManagerImpl");
}
@@ -941,7 +949,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
@Override
public boolean stop() {
_heartbeatScheduler.shutdown();
- _executor.shutdown();
+ _apiJobExecutor.shutdown();
+ _workerJobExecutor.shutdown();
return true;
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8e27120b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java
index 0b6f7a5..b1cac3e 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobMonitor.java
@@ -115,7 +115,7 @@ public class AsyncJobMonitor extends ManagerBase {
assert (_activeTasks.get(runNumber) == null);
long threadId = Thread.currentThread().getId();
- boolean fromPoolThread = Thread.currentThread().getName().contains(AsyncJobManager.JOB_POOL_THREAD_PREFIX);
+ boolean fromPoolThread = Thread.currentThread().getName().contains(AsyncJobManager.API_JOB_POOL_THREAD_PREFIX);
ActiveTaskRecord record = new ActiveTaskRecord(jobId, threadId, fromPoolThread);
_activeTasks.put(runNumber, record);
if (fromPoolThread)
[07/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
HA manager to check target state of the VM to make decisions on whether or not to carry on a cheduled step. Fix NPE in capacity manager when releasing capacity of a stopped VM.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/4312d884
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/4312d884
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/4312d884
Branch: refs/heads/master
Commit: 4312d884629d4a9e7716da4c869433cdaa8181b8
Parents: 3d5fbe7
Author: Kelven Yang <ke...@gmail.com>
Authored: Tue Mar 4 16:43:26 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
server/src/com/cloud/capacity/CapacityManagerImpl.java | 5 +++++
server/src/com/cloud/ha/HighAvailabilityManagerImpl.java | 2 ++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4312d884/server/src/com/cloud/capacity/CapacityManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/capacity/CapacityManagerImpl.java b/server/src/com/cloud/capacity/CapacityManagerImpl.java
index 957a5d0..69e8cfa 100755
--- a/server/src/com/cloud/capacity/CapacityManagerImpl.java
+++ b/server/src/com/cloud/capacity/CapacityManagerImpl.java
@@ -188,6 +188,11 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
Long clusterId = null;
if (hostId != null) {
HostVO host = _hostDao.findById(hostId);
+ if (host == null) {
+ s_logger.warn("Host " + hostId + " no long exist anymore!");
+ return true;
+ }
+
clusterId = host.getClusterId();
}
if (capacityCpu == null || capacityMemory == null || svo == null) {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4312d884/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
index 3bcaf2e..8bd569a 100755
--- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -701,6 +701,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
(vm.getHostId() != null ? vm.getHostId() : "none") + " State: " + vm.getState());
return null;
}
+
_itMgr.advanceStop(vm.getUuid(), false);
s_logger.info("Stop for " + vm + " was successful");
return null;
@@ -711,6 +712,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
(vm.getHostId() != null ? vm.getHostId() : "none") + " State: " + vm.getState());
return null;
}
+
_itMgr.advanceStop(vm.getUuid(), true);
s_logger.info("Stop for " + vm + " was successful");
return null;
[10/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
Do not use row lock in sync-queue scheduling to work around mysql locking issues.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/5310e66f
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/5310e66f
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/5310e66f
Branch: refs/heads/master
Commit: 5310e66f30fbfcc025b69b50a9a717557ac9eb3c
Parents: 7fa4715
Author: Kelven Yang <ke...@gmail.com>
Authored: Thu Mar 6 16:28:29 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
api/src/com/cloud/vm/VirtualMachine.java | 2 ++
.../jobs/impl/AsyncJobManagerImpl.java | 16 ++++++++++++++
.../jobs/impl/SyncQueueManagerImpl.java | 22 ++++++++++----------
3 files changed, 29 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5310e66f/api/src/com/cloud/vm/VirtualMachine.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/vm/VirtualMachine.java b/api/src/com/cloud/vm/VirtualMachine.java
index dd11a82..b085d4a 100755
--- a/api/src/com/cloud/vm/VirtualMachine.java
+++ b/api/src/com/cloud/vm/VirtualMachine.java
@@ -119,10 +119,12 @@ public interface VirtualMachine extends RunningOn, ControlledEntity, Identity, I
s_fsm.addTransition(State.Error, VirtualMachine.Event.DestroyRequested, State.Expunging);
s_fsm.addTransition(State.Error, VirtualMachine.Event.ExpungeOperation, State.Expunging);
+ s_fsm.addTransition(State.Starting, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
s_fsm.addTransition(State.Stopping, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
s_fsm.addTransition(State.Stopped, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
s_fsm.addTransition(State.Running, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
s_fsm.addTransition(State.Migrating, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running);
+
s_fsm.addTransition(State.Starting, VirtualMachine.Event.FollowAgentPowerOffReport, State.Stopped);
s_fsm.addTransition(State.Stopping, VirtualMachine.Event.FollowAgentPowerOffReport, State.Stopped);
s_fsm.addTransition(State.Running, VirtualMachine.Event.FollowAgentPowerOffReport, State.Stopped);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5310e66f/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
index 9b9460c..49c3032 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
@@ -659,8 +659,24 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
private Runnable getHeartbeatTask() {
return new ManagedContextRunnable() {
+
@Override
protected void runInContext() {
+ GlobalLock scanLock = GlobalLock.getInternLock("AsyncJobManagerHeartbeat");
+ try {
+ if (scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) {
+ try {
+ reallyRun();
+ } finally {
+ scanLock.unlock();
+ }
+ }
+ } finally {
+ scanLock.releaseRef();
+ }
+ }
+
+ protected void reallyRun() {
try {
List<SyncQueueItemVO> l = _queueMgr.dequeueFromAny(getMsid(), MAX_ONETIME_SCHEDULE_SIZE);
if (l != null && l.size() > 0) {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5310e66f/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java
index d8e2674..5160e05 100644
--- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java
+++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java
@@ -83,8 +83,8 @@ public class SyncQueueManagerImpl extends ManagerBase implements SyncQueueManage
return Transaction.execute(new TransactionCallback<SyncQueueItemVO>() {
@Override
public SyncQueueItemVO doInTransaction(TransactionStatus status) {
- SyncQueueVO queueVO = _syncQueueDao.lockRow(queueId, true);
- if (queueVO == null) {
+ SyncQueueVO queueVO = _syncQueueDao.findById(queueId);
+ if(queueVO == null) {
s_logger.error("Sync queue(id: " + queueId + ") does not exist");
return null;
}
@@ -139,11 +139,11 @@ public class SyncQueueManagerImpl extends ManagerBase implements SyncQueueManage
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
List<SyncQueueItemVO> l = _syncQueueItemDao.getNextQueueItems(maxItems);
- if (l != null && l.size() > 0) {
- for (SyncQueueItemVO item : l) {
- SyncQueueVO queueVO = _syncQueueDao.lockRow(item.getQueueId(), true);
- SyncQueueItemVO itemVO = _syncQueueItemDao.lockRow(item.getId(), true);
- if (queueReadyToProcess(queueVO) && itemVO.getLastProcessNumber() == null) {
+ if(l != null && l.size() > 0) {
+ for(SyncQueueItemVO item : l) {
+ SyncQueueVO queueVO = _syncQueueDao.findById(item.getQueueId());
+ SyncQueueItemVO itemVO = _syncQueueItemDao.findById(item.getId());
+ if(queueReadyToProcess(queueVO) && itemVO.getLastProcessNumber() == null) {
Long processNumber = queueVO.getLastProcessNumber();
if (processNumber == null)
processNumber = new Long(1);
@@ -184,8 +184,8 @@ public class SyncQueueManagerImpl extends ManagerBase implements SyncQueueManage
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId);
- if (itemVO != null) {
- SyncQueueVO queueVO = _syncQueueDao.lockRow(itemVO.getQueueId(), true);
+ if(itemVO != null) {
+ SyncQueueVO queueVO = _syncQueueDao.findById(itemVO.getQueueId());
_syncQueueItemDao.expunge(itemVO.getId());
@@ -213,8 +213,8 @@ public class SyncQueueManagerImpl extends ManagerBase implements SyncQueueManage
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
SyncQueueItemVO itemVO = _syncQueueItemDao.findById(queueItemId);
- if (itemVO != null) {
- SyncQueueVO queueVO = _syncQueueDao.lockRow(itemVO.getQueueId(), true);
+ if(itemVO != null) {
+ SyncQueueVO queueVO = _syncQueueDao.findById(itemVO.getQueueId());
itemVO.setLastProcessMsid(null);
itemVO.setLastProcessNumber(null);
[02/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
BUG-ID: CS-19196: Relax HA work termination rule on exceptions. It could cause premature HA termination due to unhandled exceptions. Reviewed-By:Anthony
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3123c30f
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3123c30f
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3123c30f
Branch: refs/heads/master
Commit: 3123c30f23ee0f089a554ec5ea50fac1c4731329
Parents: 5310e66
Author: Kelven Yang <ke...@gmail.com>
Authored: Fri Mar 7 17:15:04 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
server/src/com/cloud/ha/HighAvailabilityManagerImpl.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3123c30f/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
index 52100c8..48d998a 100755
--- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -913,8 +913,14 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
work.setDateTaken(null);
}
} catch (Exception e) {
- s_logger.error("Terminating " + work, e);
- work.setStep(Step.Error);
+ s_logger.warn("Encountered unhandled exception during HA process, reschedule retry", e);
+
+ long nextTime = (System.currentTimeMillis() >> 10) + _restartRetryInterval;
+
+ s_logger.info("Rescheduling " + work + " to try again at " + new Date(nextTime << 10));
+ work.setTimeToTry(nextTime);
+ work.setServerId(null);
+ work.setDateTaken(null);
}
_haDao.update(work.getId(), work);
} catch (final Throwable th) {
[03/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
Restore to old HA logic, but suppress VM state change on resource state report to avoid interferring HA.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/6ad245e6
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/6ad245e6
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/6ad245e6
Branch: refs/heads/master
Commit: 6ad245e6756176eb0ca374f121f30877498b9225
Parents: 4312d88
Author: Kelven Yang <ke...@gmail.com>
Authored: Wed Mar 5 17:03:02 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
.../src/com/cloud/ha/HighAvailabilityManager.java | 2 ++
.../src/com/cloud/vm/VirtualMachineManagerImpl.java | 4 ++--
server/src/com/cloud/ha/HighAvailabilityManagerImpl.java | 9 +++++++++
3 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6ad245e6/engine/components-api/src/com/cloud/ha/HighAvailabilityManager.java
----------------------------------------------------------------------
diff --git a/engine/components-api/src/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/com/cloud/ha/HighAvailabilityManager.java
index eb60f3e..ecfb6f6 100644
--- a/engine/components-api/src/com/cloud/ha/HighAvailabilityManager.java
+++ b/engine/components-api/src/com/cloud/ha/HighAvailabilityManager.java
@@ -100,6 +100,8 @@ public interface HighAvailabilityManager extends Manager {
void cancelScheduledMigrations(HostVO host);
+ boolean hasPendingHaWork(long vmId);
+
/**
* @return
*/
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6ad245e6/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
----------------------------------------------------------------------
diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
index 0ca0d30..e344001 100755
--- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
+++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
@@ -4100,7 +4100,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
List<VmWorkJobVO> pendingWorkJobs = _workJobDao.listPendingWorkJobs(
VirtualMachine.Type.Instance, vmId);
- if (pendingWorkJobs.size() == 0) {
+ if (pendingWorkJobs.size() == 0 || _haMgr.hasPendingHaWork(vmId)) {
// there is no pending operation job
VMInstanceVO vm = _vmDao.findById(vmId);
if (vm != null) {
@@ -4125,7 +4125,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
s_logger.warn("VM " + vmId + " no longer exists when processing VM state report");
}
} else {
- s_logger.info("There is pending job working on the VM. vm id: " + vmId + ", postpone power-change report by resetting power-change counters");
+ s_logger.info("There is pending job or HA tasks working on the VM. vm id: " + vmId + ", postpone power-change report by resetting power-change counters");
// reset VM power state tracking so that we won't lost signal when VM has
// been translated to
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6ad245e6/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
index 8bd569a..29f1e7b 100755
--- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -962,6 +962,9 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
if (oldState == State.Running && event == VirtualMachine.Event.FollowAgentPowerOffReport && newState == State.Stopped) {
final VMInstanceVO vm = _instanceDao.findById(vo.getId());
if (vm.isHaEnabled()) {
+ if (vm.getState() == State.Stopped)
+ s_logger.warn("Sanity check failed. postStateTransitionEvent reports transited to Stopped but VM " + vm + " is still at state " + vm.getState());
+
s_logger.info("Detected out-of-band stop of a HA enabled VM " + vm.getInstanceName() + ", will schedule restart");
_executor.submit(new ManagedContextRunnable() {
@Override
@@ -977,4 +980,10 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
}
return true;
}
+
+ @Override
+ public boolean hasPendingHaWork(long vmId) {
+ List<HaWorkVO> haWorks = _haDao.listRunningHaWorkForVm(vmId);
+ return haWorks.size() > 0;
+ }
}
[04/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
Send StopCommand(Cleanup) if out-of-band power-off VM is detected.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/53f306e8
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/53f306e8
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/53f306e8
Branch: refs/heads/master
Commit: 53f306e8e20023eace74c3644a20be4066ca7ca9
Parents: 8ff9460
Author: Kelven Yang <ke...@gmail.com>
Authored: Tue Mar 4 15:09:07 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
.../src/com/cloud/vm/VirtualMachineManagerImpl.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/53f306e8/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
----------------------------------------------------------------------
diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
index 2cbde35..0ca0d30 100755
--- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
+++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java
@@ -4221,6 +4221,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
case Stopped:
case Migrating:
s_logger.info("VM " + vm.getInstanceName() + " is at " + vm.getState() + " and we received a power-off report while there is no pending jobs on it");
+ VirtualMachineGuru vmGuru = getVmGuru(vm);
+ VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
+ sendStop(vmGuru, profile, true);
+
try {
stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, null);
} catch (NoTransitionException e) {
@@ -4233,9 +4237,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
s_logger.info("VM " + vm.getInstanceName() + " is sync-ed to at Stopped state according to power-off report from hypervisor");
- VirtualMachineGuru vmGuru = getVmGuru(vm);
- VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
- sendStop(vmGuru, profile, true);
break;
case Destroyed:
[08/10] git commit: updated refs/heads/master to 42d6373
Posted by ke...@apache.org.
perform VM HA based on agent state report.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/7fa4715b
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/7fa4715b
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/7fa4715b
Branch: refs/heads/master
Commit: 7fa4715b996fe551c09423b22bb24b22f9926044
Parents: 8e27120
Author: Kelven Yang <ke...@gmail.com>
Authored: Thu Mar 6 11:45:13 2014 -0800
Committer: Kelven Yang <ke...@gmail.com>
Committed: Thu Mar 13 16:59:56 2014 -0700
----------------------------------------------------------------------
.../src/com/cloud/ha/HighAvailabilityManagerImpl.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/7fa4715b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
index 29f1e7b..52100c8 100755
--- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
+++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java
@@ -393,8 +393,12 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
}
}
- HaWorkVO work =
- new HaWorkVO(vm.getId(), vm.getType(), WorkType.HA, investigate ? Step.Investigating : Step.Scheduled, hostId, vm.getState(), maxRetries + 1, vm.getUpdated());
+ if (hostId == null) {
+ hostId = vm.getLastHostId();
+ }
+
+ HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.HA, investigate ? Step.Investigating : Step.Scheduled,
+ hostId != null ? hostId : 0L, vm.getState(), maxRetries + 1, vm.getUpdated());
_haDao.persist(work);
if (s_logger.isInfoEnabled()) {
@@ -962,7 +966,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
if (oldState == State.Running && event == VirtualMachine.Event.FollowAgentPowerOffReport && newState == State.Stopped) {
final VMInstanceVO vm = _instanceDao.findById(vo.getId());
if (vm.isHaEnabled()) {
- if (vm.getState() == State.Stopped)
+ if (vm.getState() != State.Stopped)
s_logger.warn("Sanity check failed. postStateTransitionEvent reports transited to Stopped but VM " + vm + " is still at state " + vm.getState());
s_logger.info("Detected out-of-band stop of a HA enabled VM " + vm.getInstanceName() + ", will schedule restart");
@@ -970,7 +974,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai
@Override
protected void runInContext() {
try {
- scheduleRestart(vm, true);
+ scheduleRestart(vm, false);
} catch (Exception e) {
s_logger.warn("Unexpected exception when scheduling a HA restart", e);
}