You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by mc...@apache.org on 2013/11/02 02:07:43 UTC

[2/2] git commit: updated refs/heads/master to 24af282

CLOUDSTACK-5017: If SSVM is unavailable DownloadCommands will be routed
to mgmt server.


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/6eecb0b3
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/6eecb0b3
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/6eecb0b3

Branch: refs/heads/master
Commit: 6eecb0b3b5c3ea427581e2f440751b298cbb1e28
Parents: c2e2f6c
Author: Min Chen <mi...@citrix.com>
Authored: Fri Nov 1 16:14:15 2013 -0700
Committer: Min Chen <mi...@citrix.com>
Committed: Fri Nov 1 18:07:00 2013 -0700

----------------------------------------------------------------------
 .../motion/AncientDataMotionStrategy.java       |  92 +++++++++++----
 .../storage/image/TemplateServiceImpl.java      |  18 ++-
 .../endpoint/DefaultEndPointSelector.java       |  23 ++--
 .../storage/helper/HypervisorHelperImpl.java    |  21 +++-
 .../storage/image/BaseImageStoreDriverImpl.java |  45 ++++---
 .../storage/volume/VolumeServiceImpl.java       |  97 ++++++++-------
 .../driver/SimulatorImageStoreDriverImpl.java   |  34 ++++--
 .../driver/CloudStackImageStoreDriverImpl.java  |  16 ++-
 .../driver/SwiftImageStoreDriverImpl.java       |  30 ++---
 .../CloudStackPrimaryDataStoreDriverImpl.java   |  16 ++-
 .../SamplePrimaryDataStoreDriverImpl.java       |   9 +-
 .../storage/download/DownloadMonitorImpl.java   |  15 ++-
 .../storage/snapshot/SnapshotManagerImpl.java   |  17 ++-
 .../cloud/storage/upload/UploadListener.java    |   7 +-
 .../cloud/storage/upload/UploadMonitorImpl.java | 118 +++++++++++--------
 .../com/cloud/template/TemplateManagerImpl.java |   9 +-
 16 files changed, 369 insertions(+), 198 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
----------------------------------------------------------------------
diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
index a451ca4..67cc324 100644
--- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
+++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
@@ -25,7 +25,6 @@ import javax.inject.Inject;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
-import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy;
@@ -46,10 +45,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.storage.command.CopyCommand;
-import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
-import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
 
 import com.cloud.agent.api.Answer;
@@ -62,19 +57,11 @@ import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.configuration.Config;
 import com.cloud.host.Host;
-import com.cloud.host.dao.HostDao;
 import com.cloud.server.ManagementService;
 import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VolumeVO;
-import com.cloud.storage.dao.DiskOfferingDao;
-import com.cloud.storage.dao.SnapshotDao;
-import com.cloud.storage.dao.VMTemplateDao;
-import com.cloud.storage.dao.VMTemplatePoolDao;
 import com.cloud.storage.dao.VolumeDao;
-import com.cloud.storage.snapshot.SnapshotManager;
-import com.cloud.template.TemplateManager;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -177,7 +164,13 @@ AncientDataMotionStrategy implements DataMotionStrategy {
 
             CopyCommand cmd = new CopyCommand(srcForCopy.getTO(), destData.getTO(), _primaryStorageDownloadWait, _mgmtServer.getExecuteInSequence());
             EndPoint ep = selector.select(srcForCopy, destData);
-            answer = ep.sendMessage(cmd);
+            if (ep == null) {
+                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                s_logger.error(errMsg);
+                answer = new Answer(cmd, false, errMsg);
+            } else {
+                answer = ep.sendMessage(cmd);
+            }
 
             if (cacheData != null) {
                 if (srcData.getType() == DataObjectType.VOLUME && destData.getType() == DataObjectType.VOLUME) {
@@ -255,7 +248,14 @@ AncientDataMotionStrategy implements DataMotionStrategy {
             }
 
             CopyCommand cmd = new CopyCommand(srcData.getTO(), volObj.getTO(), _createVolumeFromSnapshotWait, _mgmtServer.getExecuteInSequence());
-            Answer answer = ep.sendMessage(cmd);
+            Answer answer = null;
+            if (ep == null) {
+                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                s_logger.error(errMsg);
+                answer = new Answer(cmd, false, errMsg);
+            } else {
+                answer = ep.sendMessage(cmd);
+            }
 
             return answer;
         } catch (Exception e) {
@@ -273,7 +273,14 @@ AncientDataMotionStrategy implements DataMotionStrategy {
         CopyCommand cmd = new CopyCommand(template.getTO(), volume.getTO(), 0, _mgmtServer.getExecuteInSequence());
         try {
             EndPoint ep = selector.select(volume.getDataStore());
-            Answer answer = ep.sendMessage(cmd);
+            Answer answer = null;
+            if (ep == null) {
+                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                s_logger.error(errMsg);
+                answer = new Answer(cmd, false, errMsg);
+            } else {
+                answer = ep.sendMessage(cmd);
+            }
             return answer;
         } catch (Exception e) {
             s_logger.debug("Failed to send to storage pool", e);
@@ -315,7 +322,13 @@ AncientDataMotionStrategy implements DataMotionStrategy {
 
             CopyCommand cmd = new CopyCommand(objOnImageStore.getTO(), destData.getTO(), _copyvolumewait, _mgmtServer.getExecuteInSequence());
             EndPoint ep = selector.select(objOnImageStore, destData);
-            answer = ep.sendMessage(cmd);
+            if (ep == null) {
+                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                s_logger.error(errMsg);
+                answer = new Answer(cmd, false, errMsg);
+            } else {
+                answer = ep.sendMessage(cmd);
+            }
 
             if (answer == null || !answer.getResult()) {
                 if (answer != null) {
@@ -333,7 +346,14 @@ AncientDataMotionStrategy implements DataMotionStrategy {
             DataObject cacheData = cacheMgr.createCacheObject(srcData, destScope);
             CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _copyvolumewait, _mgmtServer.getExecuteInSequence());
             EndPoint ep = selector.select(cacheData, destData);
-            Answer answer = ep.sendMessage(cmd);
+            Answer answer = null;
+            if (ep == null) {
+                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                s_logger.error(errMsg);
+                answer = new Answer(cmd, false, errMsg);
+            } else {
+                answer = ep.sendMessage(cmd);
+            }
             // delete volume on cache store
             if (cacheData != null) {
                 cacheMgr.deleteCacheObject(cacheData);
@@ -348,7 +368,14 @@ AncientDataMotionStrategy implements DataMotionStrategy {
         StoragePool destPool = (StoragePool)dataStoreMgr.getDataStore(destData.getDataStore().getId(), DataStoreRole.Primary);
         MigrateVolumeCommand command = new MigrateVolumeCommand(volume.getId(), volume.getPath(), destPool);
         EndPoint ep = selector.select(volume.getDataStore());
-        MigrateVolumeAnswer answer = (MigrateVolumeAnswer) ep.sendMessage(command);
+        Answer answer = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            answer = new Answer(command, false, errMsg);
+        } else {
+            answer = ep.sendMessage(command);
+        }
 
         if (answer == null || !answer.getResult()) {
             throw new CloudRuntimeException("Failed to migrate volume " + volume + " to storage pool " + destPool);
@@ -356,7 +383,7 @@ AncientDataMotionStrategy implements DataMotionStrategy {
             // Update the volume details after migration.
             VolumeVO volumeVo = volDao.findById(volume.getId());
             Long oldPoolId = volume.getPoolId();
-            volumeVo.setPath(answer.getVolumePath());
+            volumeVo.setPath(((MigrateVolumeAnswer)answer).getVolumePath());
             volumeVo.setFolder(destPool.getPath());
             volumeVo.setPodId(destPool.getPodId());
             volumeVo.setPoolId(destPool.getId());
@@ -431,7 +458,14 @@ AncientDataMotionStrategy implements DataMotionStrategy {
         }
 
         CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _createprivatetemplatefromsnapshotwait, _mgmtServer.getExecuteInSequence());
-        Answer answer = ep.sendMessage(cmd);
+        Answer answer = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            answer = new Answer(cmd, false, errMsg);
+        } else {
+            answer = ep.sendMessage(cmd);
+        }
 
         // clean up snapshot copied to staging
         if (needCache && srcData != null) {
@@ -455,11 +489,23 @@ AncientDataMotionStrategy implements DataMotionStrategy {
                 CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait, _mgmtServer.getExecuteInSequence());
                 cmd.setCacheTO(cacheData.getTO());
                 EndPoint ep = selector.select(srcData, destData);
-                answer = ep.sendMessage(cmd);
+                if (ep == null) {
+                    String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                    s_logger.error(errMsg);
+                    answer = new Answer(cmd, false, errMsg);
+                } else {
+                    answer = ep.sendMessage(cmd);
+                }
             } else {
                 CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait, _mgmtServer.getExecuteInSequence());
                 EndPoint ep = selector.select(srcData, destData);
-                answer = ep.sendMessage(cmd);
+                if (ep == null) {
+                    String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                    s_logger.error(errMsg);
+                    answer = new Answer(cmd, false, errMsg);
+                } else {
+                    answer = ep.sendMessage(cmd);
+                }
             }
             // clean up cache entry in case of failure
             if (answer == null || !answer.getResult()) {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index ce6198d..3e3c6d8 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -441,7 +441,14 @@ public class TemplateServiceImpl implements TemplateService {
                             tmplTO.setId(tInfo.getId());
                             DeleteCommand dtCommand = new DeleteCommand(tmplTO);
                             EndPoint ep = _epSelector.select(store);
-                            Answer answer = ep.sendMessage(dtCommand);
+                            Answer answer = null;
+                            if (ep == null) {
+                                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                                s_logger.error(errMsg);
+                                answer = new Answer(dtCommand, false, errMsg);
+                            } else {
+                                answer = ep.sendMessage(dtCommand);
+                            }
                             if (answer == null || !answer.getResult()) {
                                 s_logger.info("Failed to deleted template at store: " + store.getName());
 
@@ -513,7 +520,14 @@ public class TemplateServiceImpl implements TemplateService {
     private Map<String, TemplateProp> listTemplate(DataStore ssStore) {
         ListTemplateCommand cmd = new ListTemplateCommand(ssStore.getTO());
         EndPoint ep = _epSelector.select(ssStore);
-        Answer answer = ep.sendMessage(cmd);
+        Answer answer = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            answer = new Answer(cmd, false, errMsg);
+        } else {
+            answer = ep.sendMessage(cmd);
+        }
         if (answer != null && answer.getResult()) {
             ListTemplateAnswer tanswer = (ListTemplateAnswer) answer;
             return tanswer.getTemplateInfo();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
index 196b08b..22df463 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
@@ -27,15 +27,17 @@ import java.util.List;
 
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.storage.LocalHostEndpoint;
 import org.apache.cloudstack.storage.RemoteHostEndPoint;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
 
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
@@ -43,10 +45,10 @@ import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.ScopeType;
+import com.cloud.storage.Storage.TemplateType;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchCriteria.Op;
-import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.exception.CloudRuntimeException;
 
@@ -209,9 +211,7 @@ public class DefaultEndPointSelector implements EndPointSelector {
         // we can arbitrarily pick one ssvm to do that task
         List<HostVO> ssAHosts = listUpAndConnectingSecondaryStorageVmHost(dcId);
         if (ssAHosts == null || ssAHosts.isEmpty()) {
-            s_logger.info("No running ssvm is found, so command will be sent to LocalHostEndPoint");
-            return LocalHostEndpoint.getEndpoint(); // use local host as endpoint in
-            // case of no ssvm existing
+            return null;
         }
         Collections.shuffle(ssAHosts);
         HostVO host = ssAHosts.get(0);
@@ -232,7 +232,16 @@ public class DefaultEndPointSelector implements EndPointSelector {
     @Override
     public EndPoint select(DataObject object) {
         DataStore store = object.getDataStore();
-        return select(store);
+        EndPoint ep = select(store);
+        if (ep != null)
+            return ep;
+        if (object instanceof TemplateInfo) {
+            TemplateInfo tmplInfo = (TemplateInfo)object;
+            if (store.getScope().getScopeType() == ScopeType.ZONE && store.getScope().getScopeId() == null && tmplInfo.getTemplateType() == TemplateType.SYSTEM) {
+                return LocalHostEndpoint.getEndpoint(); // for bootstrap system vm template downloading to region image store
+            }
+        }
+        return null;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/engine/storage/src/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java b/engine/storage/src/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java
index 86c90a1..7dbd35c 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/helper/HypervisorHelperImpl.java
@@ -23,6 +23,8 @@ import java.util.UUID;
 
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
@@ -32,7 +34,6 @@ import org.apache.cloudstack.storage.command.IntroduceObjectAnswer;
 import org.apache.cloudstack.storage.command.IntroduceObjectCmd;
 import org.apache.cloudstack.storage.vmsnapshot.VMSnapshotHelper;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
@@ -66,7 +67,14 @@ public class HypervisorHelperImpl implements HypervisorHelper {
     public DataTO introduceObject(DataTO object, Scope scope, Long storeId) {
         EndPoint ep = selector.select(scope, storeId);
         IntroduceObjectCmd cmd = new IntroduceObjectCmd(object);
-        Answer answer = ep.sendMessage(cmd);
+        Answer answer = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            answer = new Answer(cmd, false, errMsg);
+        } else {
+            answer = ep.sendMessage(cmd);
+        }
         if (answer == null || !answer.getResult()) {
             String errMsg = answer == null ? null : answer.getDetails();
             throw new CloudRuntimeException("Failed to introduce object, due to " + errMsg);
@@ -79,7 +87,14 @@ public class HypervisorHelperImpl implements HypervisorHelper {
     public boolean forgetObject(DataTO object, Scope scope, Long storeId) {
         EndPoint ep = selector.select(scope, storeId);
         ForgetObjectCmd cmd = new ForgetObjectCmd(object);
-        Answer answer = ep.sendMessage(cmd);
+        Answer answer = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            answer = new Answer(cmd, false, errMsg);
+        } else {
+            answer = ep.sendMessage(cmd);
+        }
         if (answer == null || !answer.getResult()) {
             String errMsg = answer == null ? null : answer.getDetails();
             if (errMsg != null) {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index 3a70d8f..bd93e73 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -18,17 +18,13 @@
  */
 package org.apache.cloudstack.storage.image;
 
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.storage.Proxy;
-import com.cloud.agent.api.to.DataObjectType;
-import com.cloud.agent.api.to.DataTO;
-import com.cloud.storage.VMTemplateStorageResourceAssoc;
-import com.cloud.storage.VMTemplateVO;
-import com.cloud.storage.VolumeVO;
-import com.cloud.storage.dao.VMTemplateDao;
-import com.cloud.storage.dao.VolumeDao;
-import com.cloud.storage.download.DownloadMonitor;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Date;
+
+import javax.inject.Inject;
+
+import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
@@ -47,13 +43,17 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 
-import org.apache.log4j.Logger;
-
-import javax.inject.Inject;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Date;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.Proxy;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.DataTO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.storage.download.DownloadMonitor;
 
 public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
     private static final Logger s_logger = Logger.getLogger(BaseImageStoreDriverImpl.class);
@@ -239,7 +239,14 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
         try {
             DeleteCommand cmd = new DeleteCommand(data.getTO());
             EndPoint ep = _epSelector.select(data);
-            Answer answer = ep.sendMessage(cmd);
+            Answer answer = null;
+            if (ep == null) {
+                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                s_logger.error(errMsg);
+                answer = new Answer(cmd, false, errMsg);
+            } else {
+                answer = ep.sendMessage(cmd);
+            }
             if (answer != null && !answer.getResult()) {
                 result.setResult(answer.getDetails());
             }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
index 2337858..79e8cc8 100644
--- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
+++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
@@ -18,37 +18,16 @@
  */
 package org.apache.cloudstack.storage.volume;
 
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.storage.ListVolumeAnswer;
-import com.cloud.agent.api.storage.ListVolumeCommand;
-import com.cloud.agent.api.to.VirtualMachineTO;
-import com.cloud.alert.AlertManager;
-import com.cloud.configuration.Config;
-import com.cloud.configuration.Resource.ResourceType;
-import com.cloud.event.EventTypes;
-import com.cloud.event.UsageEventUtils;
-import com.cloud.exception.ConcurrentOperationException;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.host.Host;
-import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.ScopeType;
-import com.cloud.storage.StoragePool;
-import com.cloud.storage.VMTemplateStoragePoolVO;
-import com.cloud.storage.VMTemplateStorageResourceAssoc;
-import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
-import com.cloud.storage.Volume;
-import com.cloud.storage.Volume.State;
-import com.cloud.storage.VolumeVO;
-import com.cloud.storage.dao.VMTemplatePoolDao;
-import com.cloud.storage.dao.VolumeDao;
-import com.cloud.storage.snapshot.SnapshotManager;
-import com.cloud.storage.template.TemplateProp;
-import com.cloud.user.AccountManager;
-import com.cloud.user.ResourceLimitService;
-import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.db.DB;
-import com.cloud.utils.db.GlobalLock;
-import com.cloud.utils.exception.CloudRuntimeException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
@@ -81,14 +60,38 @@ import org.apache.cloudstack.storage.datastore.PrimaryDataStoreProviderManager;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
 
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.storage.ListVolumeAnswer;
+import com.cloud.agent.api.storage.ListVolumeCommand;
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.alert.AlertManager;
+import com.cloud.configuration.Config;
+import com.cloud.configuration.Resource.ResourceType;
+import com.cloud.event.EventTypes;
+import com.cloud.event.UsageEventUtils;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.host.Host;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.ScopeType;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.Volume;
+import com.cloud.storage.Volume.State;
+import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.storage.dao.VolumeDao;
+import com.cloud.storage.snapshot.SnapshotManager;
+import com.cloud.storage.template.TemplateProp;
+import com.cloud.user.AccountManager;
+import com.cloud.user.ResourceLimitService;
+import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.db.DB;
+import com.cloud.utils.db.GlobalLock;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 @Component
 public class VolumeServiceImpl implements VolumeService {
@@ -1255,7 +1258,14 @@ public class VolumeServiceImpl implements VolumeService {
                         tmplTO.setId(tInfo.getId());
                         DeleteCommand dtCommand = new DeleteCommand(tmplTO);
                         EndPoint ep = _epSelector.select(store);
-                        Answer answer = ep.sendMessage(dtCommand);
+                        Answer answer = null;
+                        if (ep == null) {
+                            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                            s_logger.error(errMsg);
+                            answer = new Answer(dtCommand, false, errMsg);
+                        } else {
+                            answer = ep.sendMessage(dtCommand);
+                        }
                         if (answer == null || !answer.getResult()) {
                             s_logger.info("Failed to deleted volume at store: " + store.getName());
 
@@ -1280,7 +1290,14 @@ public class VolumeServiceImpl implements VolumeService {
     private Map<Long, TemplateProp> listVolume(DataStore store) {
         ListVolumeCommand cmd = new ListVolumeCommand(store.getTO(), store.getUri());
         EndPoint ep = _epSelector.select(store);
-        Answer answer = ep.sendMessage(cmd);
+        Answer answer = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            answer = new Answer(cmd, false, errMsg);
+        } else {
+            answer = ep.sendMessage(cmd);
+        }
         if (answer != null && answer.getResult()) {
             ListVolumeAnswer tanswer = (ListVolumeAnswer) answer;
             return tanswer.getTemplateInfo();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java b/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java
index e18f8e5..26f8a40 100644
--- a/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java
+++ b/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/driver/SimulatorImageStoreDriverImpl.java
@@ -20,14 +20,12 @@
 package org.apache.cloudstack.storage.datastore.driver;
 
 
-import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.to.DataObjectType;
-import com.cloud.agent.api.to.DataStoreTO;
-import com.cloud.agent.api.to.NfsTO;
-import com.cloud.storage.Storage;
-import com.cloud.storage.VMTemplateStorageResourceAssoc;
-import com.cloud.storage.dao.VMTemplateDao;
-import com.cloud.storage.dao.VolumeDao;
+import java.util.UUID;
+
+import javax.inject.Inject;
+
+import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -39,10 +37,15 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.image.BaseImageStoreDriverImpl;
 import org.apache.cloudstack.storage.image.store.ImageStoreImpl;
-import org.apache.log4j.Logger;
 
-import javax.inject.Inject;
-import java.util.UUID;
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.storage.Storage;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VolumeDao;
 
 public class SimulatorImageStoreDriverImpl extends BaseImageStoreDriverImpl {
     private static final Logger s_logger = Logger.getLogger(SimulatorImageStoreDriverImpl.class);
@@ -71,9 +74,9 @@ public class SimulatorImageStoreDriverImpl extends BaseImageStoreDriverImpl {
     @Override
     public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback<CreateCmdResult> callback) {
         if (data.getType() == DataObjectType.TEMPLATE) {
-            this.createTemplate(data, callback);
+            createTemplate(data, callback);
         } else if (data.getType() == DataObjectType.VOLUME) {
-            this.createVolume(data, callback);
+            createVolume(data, callback);
         }
     }
 
@@ -108,6 +111,11 @@ public class SimulatorImageStoreDriverImpl extends BaseImageStoreDriverImpl {
     @Override
     public String createEntityExtractUrl(DataStore store, String installPath, Storage.ImageFormat format, DataObject dataObject) {
         EndPoint ep = _epSelector.select(store);
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            return null;
+        }
         // Create Symlink at ssvm
         String path = installPath;
         String uuid = UUID.randomUUID().toString() + "." + format.getFileExtension();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
index eefa352..be7c77d 100644
--- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
+++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
@@ -22,18 +22,17 @@ import java.util.UUID;
 
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.storage.image.BaseImageStoreDriverImpl;
 import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
 import org.apache.cloudstack.storage.image.store.ImageStoreImpl;
 
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.storage.image.BaseImageStoreDriverImpl;
-
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
 import com.cloud.agent.api.to.DataStoreTO;
@@ -68,7 +67,14 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl {
         String path = installPath;
         String uuid = UUID.randomUUID().toString() + "." + format.getFileExtension();
         CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity) store).getMountPoint(), path, uuid, dataObject.getTO());
-        Answer ans = ep.sendMessage(cmd);
+        Answer ans = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            ans = new Answer(cmd, false, errMsg);
+        } else {
+            ans = ep.sendMessage(cmd);
+        }
         if (ans == null || !ans.getResult()) {
             String errorString = "Unable to create a link for entity at " + installPath + " on ssvm," + ans.getDetails();
             s_logger.error(errorString);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
index 4a95844..f8c0f05 100644
--- a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
+++ b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
@@ -19,32 +19,34 @@
 package org.apache.cloudstack.storage.datastore.driver;
 
 import java.util.Map;
-import java.util.Timer;
+
 import javax.inject.Inject;
 
-import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.to.DataObjectType;
-import com.cloud.storage.download.DownloadListener;
-import com.cloud.storage.template.TemplateConstants;
-import com.cloud.storage.upload.UploadListener;
-import com.cloud.template.VirtualMachineTemplate;
-import com.cloud.utils.component.ComponentContext;
+import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.engine.subsystem.api.storage.*;
+import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
+import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
+import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
 import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.command.DownloadProgressCommand;
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
 import org.apache.cloudstack.storage.image.BaseImageStoreDriverImpl;
 import org.apache.cloudstack.storage.image.store.ImageStoreImpl;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
-import org.apache.log4j.Logger;
 
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.to.DataObjectType;
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.SwiftTO;
 import com.cloud.exception.UnsupportedServiceException;
 import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
     private static final Logger s_logger = Logger.getLogger(SwiftImageStoreDriverImpl.class);
@@ -80,8 +82,9 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
 
         EndPoint ep = _epSelector.select(data);
         if (ep == null) {
-            s_logger.warn("There is no secondary storage VM for downloading template to image store " + dataStore.getName());
-            return;
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            throw new CloudRuntimeException(errMsg);
         }
 
         CreateContext<CreateCmdResult> context = new CreateContext<CreateCmdResult>(callback, data);
@@ -96,7 +99,6 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
         }
         ep.sendMessageAsync(dcmd, caller);
 
-
     }
 
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java
index 82dc347..2db0a08 100644
--- a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java
+++ b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java
@@ -22,6 +22,8 @@ import java.util.UUID;
 
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
@@ -45,7 +47,6 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.volume.VolumeObject;
-import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.ResizeVolumeAnswer;
@@ -214,7 +215,14 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri
 
                 CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _primaryStorageDownloadWait, true);
                 EndPoint ep = epSelector.select(srcData, destData);
-                Answer answer = ep.sendMessage(cmd);
+                Answer answer = null;
+                if (ep == null) {
+                    String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                    s_logger.error(errMsg);
+                    answer = new Answer(cmd, false, errMsg);
+                } else {
+                    answer = ep.sendMessage(cmd);
+                }
                 CopyCommandResult result = new CopyCommandResult("", answer);
                 callback.complete(result);
             }
@@ -245,7 +253,7 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri
 
 
             CreateObjectCommand cmd = new CreateObjectCommand(snapshotTO);
-            EndPoint ep = this.epSelector.select(snapshot);
+            EndPoint ep = epSelector.select(snapshot);
             Answer answer = null;
             if ( ep == null ){
                 String errMsg = "No remote endpoint to send DeleteCommand, check if host or ssvm is down?";
@@ -284,7 +292,7 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri
                 resizeParameter.newSize, resizeParameter.shrinkOk, resizeParameter.instanceName);
         CreateCmdResult result = new CreateCmdResult(null, null);
         try {
-            ResizeVolumeAnswer answer = (ResizeVolumeAnswer) this.storageMgr.sendToPool(pool, resizeParameter.hosts,
+            ResizeVolumeAnswer answer = (ResizeVolumeAnswer) storageMgr.sendToPool(pool, resizeParameter.hosts,
                     resizeCmd);
             if (answer != null && answer.getResult()) {
                 long finalSize = answer.getNewSize();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/driver/SamplePrimaryDataStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/driver/SamplePrimaryDataStoreDriverImpl.java b/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/driver/SamplePrimaryDataStoreDriverImpl.java
index 75e8823..8f4c7bb 100644
--- a/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/driver/SamplePrimaryDataStoreDriverImpl.java
+++ b/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/driver/SamplePrimaryDataStoreDriverImpl.java
@@ -18,6 +18,8 @@ package org.apache.cloudstack.storage.datastore.driver;
 
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
@@ -34,12 +36,12 @@ import org.apache.cloudstack.framework.async.AsyncRpcContext;
 import org.apache.cloudstack.storage.command.CommandResult;
 import org.apache.cloudstack.storage.command.CreateObjectCommand;
 import org.apache.cloudstack.storage.datastore.DataObjectManager;
-import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 public class SamplePrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver {
     private static final Logger s_logger = Logger.getLogger(SamplePrimaryDataStoreDriverImpl.class);
@@ -163,6 +165,11 @@ public class SamplePrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver
     @Override
     public void createAsync(DataStore dataStore, DataObject vol, AsyncCompletionCallback<CreateCmdResult> callback) {
         EndPoint ep = selector.select(vol);
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            throw new CloudRuntimeException(errMsg);
+        }
         CreateObjectCommand createCmd = new CreateObjectCommand(null);
 
         CreateVolumeContext<CreateCmdResult> context = new CreateVolumeContext<CreateCmdResult>(callback, vol);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
index eb790a4..d455f1e 100755
--- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
+++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
@@ -22,11 +22,13 @@ import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Timer;
-import java.util.concurrent.ConcurrentHashMap;
 
 import javax.ejb.Local;
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -48,16 +50,13 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
 
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.storage.DownloadAnswer;
 import com.cloud.agent.api.storage.Proxy;
 import com.cloud.configuration.Config;
 import com.cloud.storage.RegisterVolumePayload;
-import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.storage.Volume;
 import com.cloud.storage.dao.VMTemplateDao;
@@ -159,7 +158,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
         Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes();
         if (vmTemplateStore != null) {
             start();
-            VirtualMachineTemplate tmpl = this._templateDao.findById(template.getId());
+            VirtualMachineTemplate tmpl = _templateDao.findById(template.getId());
             DownloadCommand dcmd = new DownloadCommand((TemplateObjectTO)(template.getTO()), maxTemplateSizeInBytes);
             dcmd.setProxy(getHttpProxy());
             if (downloadJobExists) {
@@ -239,7 +238,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
         Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes();
         if (volumeHost != null) {
             start();
-            Volume vol = this._volumeDao.findById(volume.getId());
+            Volume vol = _volumeDao.findById(volume.getId());
             DownloadCommand dcmd = new DownloadCommand((VolumeObjectTO)(volume.getTO()), maxVolumeSizeInBytes, checkSum, url, format);
             dcmd.setProxy(getHttpProxy());
             if (downloadJobExists) {
@@ -247,7 +246,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
                 dcmd.setResourceType(ResourceType.VOLUME);
             }
 
-            EndPoint ep = this._epSelector.select(volume);
+            EndPoint ep = _epSelector.select(volume);
             if (ep == null) {
                 s_logger.warn("There is no secondary storage VM for image store " + store.getName());
                 return;

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
index d15393c..9888c76 100755
--- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
+++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
@@ -26,6 +26,9 @@ import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotPolicyCmd;
 import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd;
 import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd;
@@ -41,10 +44,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
-import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
 import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotStrategy.SnapshotOperation;
-import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
@@ -53,8 +53,6 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
@@ -593,7 +591,14 @@ public class SnapshotManagerImpl extends ManagerBase implements SnapshotManager,
                 String snapshotDir = TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/" + accountId + "/" + volumeId;
                 DeleteSnapshotsDirCommand cmd = new DeleteSnapshotsDirCommand(ssHost.getTO(), snapshotDir);
                 EndPoint ep = _epSelector.select(ssHost);
-                Answer answer = ep.sendMessage(cmd);
+                Answer answer = null;
+                if (ep == null) {
+                    String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                    s_logger.error(errMsg);
+                    answer = new Answer(cmd, false, errMsg);
+                } else {
+                    answer = ep.sendMessage(cmd);
+                }
                 if ((answer != null) && answer.getResult()) {
                     s_logger.debug("Deleted all snapshots for volume: " + volumeId + " under account: " + accountId);
                 } else {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/server/src/com/cloud/storage/upload/UploadListener.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/upload/UploadListener.java b/server/src/com/cloud/storage/upload/UploadListener.java
index add5877..f3b6806 100755
--- a/server/src/com/cloud/storage/upload/UploadListener.java
+++ b/server/src/com/cloud/storage/upload/UploadListener.java
@@ -21,12 +21,12 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Timer;
-import java.util.TimerTask;
 
 import javax.inject.Inject;
 
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd;
 import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd;
 import org.apache.cloudstack.api.command.user.volume.ExtractVolumeCmd;
@@ -437,6 +437,11 @@ public class UploadListener implements Listener {
             }
             try {
                 EndPoint ep = _epSelector.select(sserver);
+                if (ep == null) {
+                    String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                    s_logger.error(errMsg);
+                    return;
+                }
                 ep.sendMessageAsync(new UploadProgressCommand(getCommand(), getJobId(), reqType), new Callback(ep.getId(), this));
             } catch (Exception e) {
                 s_logger.debug("Send command failed", e);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/server/src/com/cloud/storage/upload/UploadMonitorImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java
index 4eb4900..4785e49 100755
--- a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java
+++ b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java
@@ -34,6 +34,7 @@ import javax.naming.ConfigurationException;
 
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -167,6 +168,11 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor {
 
 		try {
 		    EndPoint ep = _epSelector.select(secStore);
+            if (ep == null) {
+                String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                s_logger.error(errMsg);
+                return;
+            }
             ep.sendMessageAsync(ucmd, new UploadListener.Callback(ep.getId(), ul));
         } catch (Exception e) {
 			s_logger.warn("Unable to start upload of volume " + volume.getName() + " from " + secStore.getName() + " to " +url, e);
@@ -194,6 +200,11 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor {
 			_listenerMap.put(uploadTemplateObj, ul);
 			try{
 			    EndPoint ep = _epSelector.select(secStore);
+                if (ep == null) {
+                    String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                    s_logger.error(errMsg);
+                    return null;
+                }
                 ep.sendMessageAsync(ucmd, new UploadListener.Callback(ep.getId(), ul));
             } catch (Exception e) {
 				s_logger.warn("Unable to start upload of " + template.getUniqueName() + " from " + secStore.getName() + " to " +url, e);
@@ -205,61 +216,66 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor {
 		return null;
 	}
 
-	@Override
-	public UploadVO createEntityDownloadURL(VMTemplateVO template, TemplateDataStoreVO vmTemplateHost, Long dataCenterId, long eventId) {
+    @Override
+    public UploadVO createEntityDownloadURL(VMTemplateVO template, TemplateDataStoreVO vmTemplateHost, Long dataCenterId, long eventId) {
 
-	    String errorString = "";
-	    boolean success = false;
-	    Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE ;
-
-            // find an endpoint to send command
-            DataStore store = storeMgr.getDataStore(vmTemplateHost.getDataStoreId(), DataStoreRole.Image);
-            EndPoint ep = _epSelector.select(store);
-
-	    //Check if it already exists.
-	    List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(template.getId(), type, UploadVO.Status.DOWNLOAD_URL_CREATED);
-	    if (extractURLList.size() > 0) {
-               // do some check here
-               UploadVO upload = extractURLList.get(0);
-               String uploadUrl = extractURLList.get(0).getUploadUrl();
-               String[] token = uploadUrl.split("/");
-               // example: uploadUrl = https://10-11-101-112.realhostip.com/userdata/2fdd9a70-9c4a-4a04-b1d5-1e41c221a1f9.iso
-               // then token[2] = 10-11-101-112.realhostip.com, token[4] = 2fdd9a70-9c4a-4a04-b1d5-1e41c221a1f9.iso
-               String hostname = ep.getPublicAddr().replace(".", "-") + ".";
-               if ((token != null) && (token.length == 5) && (token[2].equals(hostname + _ssvmUrlDomain))) // ssvm publicip and domain suffix not changed
-                   return extractURLList.get(0);
-               else if ((token != null) && (token.length == 5) && (token[2].startsWith(hostname))) { // domain suffix changed
-                   String uuid = token[4];
-                   uploadUrl = generateCopyUrl(ep.getPublicAddr(), uuid);
-                   UploadVO vo = _uploadDao.createForUpdate();
-                   vo.setLastUpdated(new Date());
-                   vo.setUploadUrl(uploadUrl);
-                   _uploadDao.update(upload.getId(), vo);
-                   return _uploadDao.findById(upload.getId(), true);
-               } else { // ssvm publicip changed
-                   return null;
-               }
+        String errorString = "";
+        boolean success = false;
+        Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE;
+
+        // find an endpoint to send command
+        DataStore store = storeMgr.getDataStore(vmTemplateHost.getDataStoreId(), DataStoreRole.Image);
+        EndPoint ep = _epSelector.select(store);
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            return null;
+        }
+
+        //Check if it already exists.
+        List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(template.getId(), type, UploadVO.Status.DOWNLOAD_URL_CREATED);
+        if (extractURLList.size() > 0) {
+            // do some check here
+            UploadVO upload = extractURLList.get(0);
+            String uploadUrl = extractURLList.get(0).getUploadUrl();
+            String[] token = uploadUrl.split("/");
+            // example: uploadUrl = https://10-11-101-112.realhostip.com/userdata/2fdd9a70-9c4a-4a04-b1d5-1e41c221a1f9.iso
+            // then token[2] = 10-11-101-112.realhostip.com, token[4] = 2fdd9a70-9c4a-4a04-b1d5-1e41c221a1f9.iso
+            String hostname = ep.getPublicAddr().replace(".", "-") + ".";
+            if ((token != null) && (token.length == 5) && (token[2].equals(hostname + _ssvmUrlDomain))) // ssvm publicip and domain suffix not changed
+                return extractURLList.get(0);
+            else if ((token != null) && (token.length == 5) && (token[2].startsWith(hostname))) { // domain suffix changed
+                String uuid = token[4];
+                uploadUrl = generateCopyUrl(ep.getPublicAddr(), uuid);
+                UploadVO vo = _uploadDao.createForUpdate();
+                vo.setLastUpdated(new Date());
+                vo.setUploadUrl(uploadUrl);
+                _uploadDao.update(upload.getId(), vo);
+                return _uploadDao.findById(upload.getId(), true);
+            } else { // ssvm publicip changed
+                return null;
+            }
         }
 
-	    // It doesn't exist so create a DB entry.
-	    UploadVO uploadTemplateObj = new UploadVO(vmTemplateHost.getDataStoreId(), template.getId(), new Date(),
-	                                                Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD);
-	    uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath());
-	    _uploadDao.persist(uploadTemplateObj);
+        // It doesn't exist so create a DB entry.
+        UploadVO uploadTemplateObj = new UploadVO(vmTemplateHost.getDataStoreId(), template.getId(), new Date(),
+                Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD);
+        uploadTemplateObj.setInstallPath(vmTemplateHost.getInstallPath());
+        _uploadDao.persist(uploadTemplateObj);
 
-	    try{
-    	    // Create Symlink at ssvm
-	    	String path = vmTemplateHost.getInstallPath();
-	    	String uuid = UUID.randomUUID().toString() + "." + template.getFormat().getFileExtension(); // adding "." + vhd/ova... etc.
-	    	CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity)store).getMountPoint(), path, uuid, null);
-	    	Answer ans = ep.sendMessage(cmd);
-	        if (ans == null || !ans.getResult()) {
-    	        errorString = "Unable to create a link for " +type+ " id:"+template.getId() + "," + ans.getDetails();
+        try {
+            // Create Symlink at ssvm
+            String path = vmTemplateHost.getInstallPath();
+            String uuid = UUID.randomUUID().toString() + "." + template.getFormat().getFileExtension(); // adding "." + vhd/ova... etc.
+            CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity)store).getMountPoint(), path, uuid, null);
+            Answer ans = ep.sendMessage(cmd);
+            if (ans == null || !ans.getResult()) {
+                errorString = "Unable to create a link for " + type + " id:" + template.getId() + "," + ans.getDetails();
                 s_logger.error(errorString);
                 throw new CloudRuntimeException(errorString);
             }
 
-    	    //Construct actual URL locally now that the symlink exists at SSVM
+            //Construct actual URL locally now that the symlink exists at SSVM
             String extractURL = generateCopyUrl(ep.getPublicAddr(), uuid);
             UploadVO vo = _uploadDao.createForUpdate();
             vo.setLastUpdated(new Date());
@@ -268,17 +284,17 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor {
             _uploadDao.update(uploadTemplateObj.getId(), vo);
             success = true;
             return _uploadDao.findById(uploadTemplateObj.getId(), true);
-	    }finally{
-           if(!success){
+        } finally {
+            if (!success) {
                 UploadVO uploadJob = _uploadDao.createForUpdate(uploadTemplateObj.getId());
                 uploadJob.setLastUpdated(new Date());
                 uploadJob.setErrorString(errorString);
                 uploadJob.setUploadState(Status.ERROR);
                 _uploadDao.update(uploadTemplateObj.getId(), uploadJob);
             }
-	    }
+        }
 
-	}
+    }
 
 	@Override
     public void createVolumeDownloadURL(Long entityId, String path, Type type, Long dataCenterId, Long uploadId, ImageFormat format) {

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/6eecb0b3/server/src/com/cloud/template/TemplateManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java
index 50e557a..5ce499c 100755
--- a/server/src/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/com/cloud/template/TemplateManagerImpl.java
@@ -583,7 +583,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     public String getChecksum(DataStore store, String templatePath) {
         EndPoint ep = _epSelector.select(store);
         ComputeChecksumCommand cmd = new ComputeChecksumCommand(store.getTO(), templatePath);
-        Answer answer = ep.sendMessage(cmd);
+        Answer answer = null;
+        if (ep == null) {
+            String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+            s_logger.error(errMsg);
+            answer = new Answer(cmd, false, errMsg);
+        } else {
+            answer = ep.sendMessage(cmd);
+        }
         if (answer != null && answer.getResult()) {
             return answer.getDetails();
         }