You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ni...@apache.org on 2014/11/14 22:18:35 UTC
git commit: updated refs/heads/master to 3907bc6
Repository: cloudstack
Updated Branches:
refs/heads/master a49e24095 -> 3907bc6f9
CLOUDSTACK-7916: Generate Alerts if System VMs cannot be started.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3907bc6f
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3907bc6f
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3907bc6f
Branch: refs/heads/master
Commit: 3907bc6f9d2b4a61a4d0b32e5c1a15d95210531e
Parents: a49e240
Author: Nitin Mehta <ni...@citrix.com>
Authored: Fri Nov 14 13:18:27 2014 -0800
Committer: Nitin Mehta <ni...@citrix.com>
Committed: Fri Nov 14 13:18:27 2014 -0800
----------------------------------------------------------------------
.../cloud/alert/ConsoleProxyAlertAdapter.java | 12 +-
.../alert/SecondaryStorageVmAlertAdapter.java | 11 +-
.../consoleproxy/ConsoleProxyManagerImpl.java | 74 +++++++------
.../SecondaryStorageManagerImpl.java | 110 ++++++++++---------
4 files changed, 112 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java b/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
index df4f5ba..4ca5f4a 100644
--- a/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
+++ b/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java
@@ -53,7 +53,8 @@ public class ConsoleProxyAlertAdapter extends AdapterBase implements AlertAdapte
DataCenterVO dc = _dcDao.findById(args.getZoneId());
ConsoleProxyVO proxy = args.getProxy();
- if (proxy == null)
+ //FIXME - Proxy can be null in case of creation failure. Have a better fix than checking for != 0
+ if (proxy == null && args.getProxyId() != 0)
proxy = _consoleProxyDao.findById(args.getProxyId());
switch (args.getType()) {
@@ -98,12 +99,9 @@ public class ConsoleProxyAlertAdapter extends AdapterBase implements AlertAdapte
case ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE:
if (s_logger.isDebugEnabled())
- s_logger.debug("Console proxy creation failure, zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " +
- proxy.getPublicIpAddress() + ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()));
-
- _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY, args.getZoneId(), proxy.getPodIdToDeployIn(),
- "Console proxy creation failure. zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() +
- ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()) + ", error details: " + args.getMessage(),
+ s_logger.debug("Console proxy creation failure, zone: " + dc.getName());
+ _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY, args.getZoneId(), null,
+ "Console proxy creation failure. zone: " + dc.getName() + ", error details: " + args.getMessage(),
"Console proxy creation failure (zone " + dc.getName() + ")");
break;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java b/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
index b7834bb..3bfedce 100644
--- a/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
+++ b/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java
@@ -53,7 +53,7 @@ public class SecondaryStorageVmAlertAdapter extends AdapterBase implements Alert
DataCenterVO dc = _dcDao.findById(args.getZoneId());
SecondaryStorageVmVO secStorageVm = args.getSecStorageVm();
- if (secStorageVm == null)
+ if (secStorageVm == null && args.getSecStorageVmId() != 0)
secStorageVm = _ssvmDao.findById(args.getSecStorageVmId());
switch (args.getType()) {
@@ -103,13 +103,10 @@ public class SecondaryStorageVmAlertAdapter extends AdapterBase implements Alert
case SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE:
if (s_logger.isDebugEnabled())
- s_logger.debug("Secondary Storage Vm creation failure, zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " +
- secStorageVm.getPublicIpAddress() + ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()));
+ s_logger.debug("Secondary Storage Vm creation failure, zone: " + dc.getName());
- _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SSVM, args.getZoneId(), secStorageVm.getPodIdToDeployIn(),
- "Secondary Storage Vm creation failure. zone: " +
- dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + secStorageVm.getPublicIpAddress() + ", private IP: " +
- (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()) + ", error details: " + args.getMessage(),
+ _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SSVM, args.getZoneId(), null,
+ "Secondary Storage Vm creation failure. zone: " + dc.getName() + ", error details: " + args.getMessage(),
"Secondary Storage Vm creation failure (zone " + dc.getName() + ")");
break;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
index 14d782a..4ccaa98 100755
--- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
+++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java
@@ -539,6 +539,7 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
if (proxy.getState() == VirtualMachine.State.Stopped) {
_itMgr.advanceStart(proxy.getUuid(), null, null);
proxy = _consoleProxyDao.findById(proxy.getId());
+ return proxy;
}
// For VMs that are in Stopping, Starting, Migrating state, let client to wait by returning null
@@ -668,9 +669,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
if (s_logger.isDebugEnabled()) {
s_logger.debug("Unable to allocate console proxy storage, remove the console proxy record from DB, proxy id: " + proxyVmId);
}
-
- SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this,
- new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE, dataCenterId, proxyVmId, null, "Unable to allocate storage"));
}
return null;
}
@@ -887,43 +885,57 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy
}
ConsoleProxyVO proxy = null;
- if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
- try {
- proxy = assignProxyFromStoppedPool(dataCenterId);
- if (proxy == null) {
- if (s_logger.isInfoEnabled()) {
- s_logger.info("No stopped console proxy is available, need to allocate a new console proxy");
- }
+ String errorString = null;
+ try{
+ if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
+ try {
+ proxy = assignProxyFromStoppedPool(dataCenterId);
+ if (proxy == null) {
+ if (s_logger.isInfoEnabled()) {
+ s_logger.info("No stopped console proxy is available, need to allocate a new console proxy");
+ }
- try {
- proxy = startNew(dataCenterId);
- } catch (ConcurrentOperationException e) {
- s_logger.info("Concurrent Operation caught " + e);
- }
- } else {
- if (s_logger.isInfoEnabled()) {
- s_logger.info("Found a stopped console proxy, bring it up to running pool. proxy vm id : " + proxy.getId());
+ try {
+ proxy = startNew(dataCenterId);
+ } catch (ConcurrentOperationException e) {
+ s_logger.info("Concurrent Operation caught " + e);
+ }
+ } else {
+ if (s_logger.isInfoEnabled()) {
+ s_logger.info("Found a stopped console proxy, bring it up to running pool. proxy vm id : " + proxy.getId());
+ }
}
+ } finally {
+ _allocProxyLock.unlock();
+ }
+ } else {
+ if (s_logger.isInfoEnabled()) {
+ s_logger.info("Unable to acquire proxy allocation lock, skip for next time");
}
- } finally {
- _allocProxyLock.unlock();
- }
- } else {
- if (s_logger.isInfoEnabled()) {
- s_logger.info("Unable to acquire proxy allocation lock, skip for next time");
}
- }
-
- if (proxy != null) {
- long proxyVmId = proxy.getId();
- proxy = startProxy(proxyVmId);
if (proxy != null) {
- if (s_logger.isInfoEnabled()) {
- s_logger.info("Console proxy " + proxy.getHostName() + " is started");
+ long proxyVmId = proxy.getId();
+ proxy = startProxy(proxyVmId);
+
+ if (proxy != null) {
+ if (s_logger.isInfoEnabled()) {
+ s_logger.info("Console proxy " + proxy.getHostName() + " is started");
+ }
+ SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this,
+ new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_UP, dataCenterId, proxy.getId(), proxy, null));
}
}
+ }catch (Exception e){
+ errorString = e.getMessage();
+ }finally {
+ // TODO - For now put all the alerts as creation failure. Distinguish between creation vs start failure in future.
+ // Also add failure reason since startvm masks some of them.
+ if(proxy == null || proxy.getState() != State.Running)
+ SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this,
+ new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE, dataCenterId, 0l, null, errorString));
}
+
}
public boolean isZoneReady(Map<Long, ZoneHostInfo> zoneHostInfoMap, long dataCenterId) {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3907bc6f/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
index 8069050..c996c0b 100755
--- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
+++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java
@@ -506,9 +506,6 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
s_logger.debug("Unable to allocate secondary storage vm storage, remove the secondary storage vm record from DB, secondary storage vm id: " +
secStorageVmId);
}
-
- SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this,
- new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE, dataCenterId, secStorageVmId, null, "Unable to allocate storage"));
}
return null;
}
@@ -654,70 +651,83 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
}
return;
}
-
- boolean secStorageVmFromStoppedPool = false;
- SecondaryStorageVmVO secStorageVm = assignSecStorageVmFromStoppedPool(dataCenterId, role);
- if (secStorageVm == null) {
- if (s_logger.isInfoEnabled()) {
- s_logger.info("No stopped secondary storage vm is available, need to allocate a new secondary storage vm");
- }
-
- if (_allocLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
- try {
- secStorageVm = startNew(dataCenterId, role);
- for (UploadVO upload : _uploadDao.listAll()) {
- _uploadDao.expunge(upload.getId());
- }
- } finally {
- _allocLock.unlock();
- }
- } else {
+ SecondaryStorageVmVO secStorageVm = null;
+ String errorString = null;
+ try{
+ boolean secStorageVmFromStoppedPool = false;
+ secStorageVm = assignSecStorageVmFromStoppedPool(dataCenterId, role);
+ if (secStorageVm == null) {
if (s_logger.isInfoEnabled()) {
- s_logger.info("Unable to acquire synchronization lock to allocate secStorageVm resource for standby capacity, wait for next scan");
+ s_logger.info("No stopped secondary storage vm is available, need to allocate a new secondary storage vm");
}
- return;
- }
- } else {
- if (s_logger.isInfoEnabled()) {
- s_logger.info("Found a stopped secondary storage vm, bring it up to running pool. secStorageVm vm id : " + secStorageVm.getId());
- }
- secStorageVmFromStoppedPool = true;
- }
- if (secStorageVm != null) {
- long secStorageVmId = secStorageVm.getId();
- GlobalLock secStorageVmLock = GlobalLock.getInternLock(getSecStorageVmLockName(secStorageVmId));
- try {
- if (secStorageVmLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
+ if (_allocLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
try {
- secStorageVm = startSecStorageVm(secStorageVmId);
+ secStorageVm = startNew(dataCenterId, role);
+ for (UploadVO upload : _uploadDao.listAll()) {
+ _uploadDao.expunge(upload.getId());
+ }
} finally {
- secStorageVmLock.unlock();
+ _allocLock.unlock();
}
} else {
if (s_logger.isInfoEnabled()) {
- s_logger.info("Unable to acquire synchronization lock to start secStorageVm for standby capacity, secStorageVm vm id : " + secStorageVm.getId());
+ s_logger.info("Unable to acquire synchronization lock to allocate secStorageVm resource for standby capacity, wait for next scan");
}
return;
}
- } finally {
- secStorageVmLock.releaseRef();
- }
-
- if (secStorageVm == null) {
+ } else {
if (s_logger.isInfoEnabled()) {
- s_logger.info("Unable to start secondary storage vm for standby capacity, secStorageVm vm Id : " + secStorageVmId +
- ", will recycle it and start a new one");
+ s_logger.info("Found a stopped secondary storage vm, bring it up to running pool. secStorageVm vm id : " + secStorageVm.getId());
}
+ secStorageVmFromStoppedPool = true;
+ }
- if (secStorageVmFromStoppedPool) {
- destroySecStorageVm(secStorageVmId);
+ if (secStorageVm != null) {
+ long secStorageVmId = secStorageVm.getId();
+ GlobalLock secStorageVmLock = GlobalLock.getInternLock(getSecStorageVmLockName(secStorageVmId));
+ try {
+ if (secStorageVmLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) {
+ try {
+ secStorageVm = startSecStorageVm(secStorageVmId);
+ } finally {
+ secStorageVmLock.unlock();
+ }
+ } else {
+ if (s_logger.isInfoEnabled()) {
+ s_logger.info("Unable to acquire synchronization lock to start secStorageVm for standby capacity, secStorageVm vm id : " + secStorageVm.getId());
+ }
+ return;
+ }
+ } finally {
+ secStorageVmLock.releaseRef();
}
- } else {
- if (s_logger.isInfoEnabled()) {
- s_logger.info("Secondary storage vm " + secStorageVm.getHostName() + " is started");
+
+ if (secStorageVm == null) {
+ if (s_logger.isInfoEnabled()) {
+ s_logger.info("Unable to start secondary storage vm for standby capacity, secStorageVm vm Id : " + secStorageVmId +
+ ", will recycle it and start a new one");
+ }
+
+ if (secStorageVmFromStoppedPool) {
+ destroySecStorageVm(secStorageVmId);
+ }
+ } else {
+ SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this,
+ new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_UP, dataCenterId, secStorageVmId, secStorageVm, null));
+ if (s_logger.isInfoEnabled()) {
+ s_logger.info("Secondary storage vm " + secStorageVm.getHostName() + " is started");
+ }
}
}
+ }catch (Exception e){
+ errorString = e.getMessage();
+ }finally{
+ // TODO - For now put all the alerts as creation failure. Distinguish between creation vs start failure in future.
+ // Also add failure reason since startvm masks some of them.
+ if(secStorageVm == null || secStorageVm.getState() != State.Running)
+ SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this,
+ new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE, dataCenterId, 0l, null, errorString));
}
}