You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by mc...@apache.org on 2014/01/09 02:43:29 UTC

git commit: updated refs/heads/master to 3223028

Updated Branches:
  refs/heads/master 649d9d927 -> 3223028fd


CLOUDSTACK-5841:Snapshots taken before migration NFS to S3 can not be
used cross-zone.


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/3223028f
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/3223028f
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/3223028f

Branch: refs/heads/master
Commit: 3223028fd6fb5b258bbe02f0f2f6993edc23dc5c
Parents: 649d9d9
Author: Min Chen <mi...@citrix.com>
Authored: Wed Jan 8 17:43:06 2014 -0800
Committer: Min Chen <mi...@citrix.com>
Committed: Wed Jan 8 17:43:06 2014 -0800

----------------------------------------------------------------------
 .../api/storage/SnapshotDataFactory.java        |   2 +
 .../subsystem/api/storage/SnapshotService.java  |   2 +
 .../orchestration/VolumeOrchestrator.java       |  20 +++-
 .../datastore/db/SnapshotDataStoreDao.java      |   2 +
 .../storage/image/TemplateServiceImpl.java      |   3 +
 .../snapshot/SnapshotDataFactoryImpl.java       |  12 +++
 .../storage/snapshot/SnapshotObject.java        |  12 ++-
 .../storage/snapshot/SnapshotServiceImpl.java   | 107 ++++++++++++++++++-
 .../endpoint/DefaultEndPointSelector.java       |   7 +-
 .../image/db/SnapshotDataStoreDaoImpl.java      |  10 ++
 10 files changed, 164 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java
index d5255f4..59e59a6 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java
@@ -30,4 +30,6 @@ public interface SnapshotDataFactory {
     SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role);
 
     List<SnapshotInfo> listSnapshotOnCache(long snapshotId);
+
+    SnapshotInfo getReadySnapshotOnCache(long snapshotId);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java
index e953eb6..000b9ec 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java
@@ -25,4 +25,6 @@ public interface SnapshotService {
     boolean deleteSnapshot(SnapshotInfo snapshot);
 
     boolean revertSnapshot(Long snapshotId);
+
+    void syncVolumeSnapshotsToRegionStore(long volumeId, DataStore store);
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
----------------------------------------------------------------------
diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
index e7126a2..72aaf3d 100644
--- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
+++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
@@ -30,12 +30,15 @@ import java.util.concurrent.ExecutionException;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import org.apache.log4j.Logger;
+
 import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
 import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
@@ -52,7 +55,6 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
-import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.to.DataTO;
 import com.cloud.agent.api.to.DiskTO;
@@ -140,6 +142,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
     SnapshotDataFactory snapshotFactory;
     @Inject
     ConfigDepot _configDepot;
+    @Inject
+    SnapshotService _snapshotSrv;
 
     private final StateMachine2<Volume.State, Volume.Event, Volume> _volStateMachine;
     protected List<StoragePoolAllocator> _storagePoolAllocators;
@@ -149,7 +153,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
     }
 
     public void setStoragePoolAllocators(List<StoragePoolAllocator> storagePoolAllocators) {
-        this._storagePoolAllocators = storagePoolAllocators;
+        _storagePoolAllocators = storagePoolAllocators;
     }
 
     protected List<PodAllocator> _podAllocators;
@@ -159,7 +163,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
     }
 
     public void setPodAllocators(List<PodAllocator> podAllocators) {
-        this._podAllocators = podAllocators;
+        _podAllocators = podAllocators;
     }
 
     protected VolumeOrchestrator() {
@@ -326,6 +330,16 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati
         VolumeInfo vol = volFactory.getVolume(volume.getId());
         DataStore store = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary);
         SnapshotInfo snapInfo = snapshotFactory.getSnapshot(snapshot.getId(), DataStoreRole.Image);
+        // sync snapshot to region store if necessary
+        DataStore snapStore = snapInfo.getDataStore();
+        long snapVolId = snapInfo.getVolumeId();
+        try {
+            _snapshotSrv.syncVolumeSnapshotsToRegionStore(snapVolId, snapStore);
+        } catch (Exception ex) {
+            // log but ignore the sync error to avoid any potential S3 down issue, it should be sync next time
+            s_logger.warn(ex.getMessage(), ex);
+        }
+        // create volume on primary from snapshot
         AsyncCallFuture<VolumeApiResult> future = volService.createVolumeFromSnapshot(vol, store, snapInfo);
         try {
             VolumeApiResult result = future.get();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java
index 83c337d..e24c035 100644
--- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java
+++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java
@@ -49,6 +49,8 @@ public interface SnapshotDataStoreDao extends GenericDao<SnapshotDataStoreVO, Lo
     // delete the snapshot entry on primary data store to make sure that next snapshot will be full snapshot
     void deleteSnapshotRecordsOnPrimary();
 
+    SnapshotDataStoreVO findReadyOnCache(long snapshotId);
+
     List<SnapshotDataStoreVO> listOnCache(long snapshotId);
 
     void updateStoreRoleToCache(long storeId);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index 1875636..2c82914 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -689,6 +689,9 @@ public class TemplateServiceImpl implements TemplateService {
     @Override
     public void syncTemplateToRegionStore(long templateId, DataStore store) {
         if (_storeMgr.isRegionStore(store)) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Sync template " + templateId + " from cache to object store...");
+            }
             // if template is on region wide object store, check if it is really downloaded there (by checking install_path). Sync template to region
             // wide store if it is not there physically.
             TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store);

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java
index 6205fe4..16c14f3 100644
--- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java
+++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java
@@ -80,6 +80,18 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory {
     }
 
     @Override
+    public SnapshotInfo getReadySnapshotOnCache(long snapshotId) {
+        SnapshotDataStoreVO snapStore = snapshotStoreDao.findReadyOnCache(snapshotId);
+        if (snapStore != null) {
+            DataStore store = storeMgr.getDataStore(snapStore.getDataStoreId(), DataStoreRole.ImageCache);
+            return getSnapshot(snapshotId, store);
+        } else {
+            return null;
+        }
+
+    }
+
+    @Override
     public List<SnapshotInfo> listSnapshotOnCache(long snapshotId) {
         List<SnapshotDataStoreVO> cacheSnapshots = snapshotStoreDao.listOnCache(snapshotId);
         List<SnapshotInfo> snapObjs = new ArrayList<SnapshotInfo>();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java
----------------------------------------------------------------------
diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java
index c05e048..9cac20d 100644
--- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java
+++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java
@@ -78,6 +78,7 @@ public class SnapshotObject implements SnapshotInfo {
     SnapshotDataStoreDao snapshotStoreDao;
     @Inject
     StorageStrategyFactory storageStrategyFactory;
+    private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case
 
     public SnapshotObject() {
 
@@ -198,6 +199,9 @@ public class SnapshotObject implements SnapshotInfo {
 
     @Override
     public String getPath() {
+        if (installPath != null)
+            return installPath;
+
         DataObjectInStore objectInStore = objectInStoreMgr.findObject(this, getDataStore());
         if (objectInStore != null) {
             return objectInStore.getInstallPath();
@@ -205,6 +209,10 @@ public class SnapshotObject implements SnapshotInfo {
         return null;
     }
 
+    public void setPath(String installPath) {
+        this.installPath = installPath;
+    }
+
     @Override
     public String getName() {
         return snapshot.getName();
@@ -354,12 +362,12 @@ public class SnapshotObject implements SnapshotInfo {
 
     @Override
     public void addPayload(Object data) {
-        this.payload = data;
+        payload = data;
     }
 
     @Override
     public Object getPayload() {
-        return this.payload;
+        return payload;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java
index fb68958..d482e70 100644
--- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java
+++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java
@@ -17,6 +17,7 @@
 
 package org.apache.cloudstack.storage.snapshot;
 
+import java.util.List;
 import java.util.concurrent.ExecutionException;
 
 import javax.inject.Inject;
@@ -37,6 +38,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
+import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
@@ -49,6 +51,9 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
 
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.Snapshot;
+import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.dao.SnapshotDao;
+import com.cloud.storage.template.TemplateConstants;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.fsm.NoTransitionException;
 
@@ -56,13 +61,17 @@ import com.cloud.utils.fsm.NoTransitionException;
 public class SnapshotServiceImpl implements SnapshotService {
     private static final Logger s_logger = Logger.getLogger(SnapshotServiceImpl.class);
     @Inject
+    protected SnapshotDao _snapshotDao;
+    @Inject
     protected SnapshotDataStoreDao _snapshotStoreDao;
     @Inject
-    SnapshotDataFactory snapshotfactory;
+    SnapshotDataFactory _snapshotFactory;
     @Inject
     DataStoreManager dataStoreMgr;
     @Inject
     DataMotionService motionSrv;
+    @Inject
+    StorageCacheManager _cacheMgr;
 
     static private class CreateSnapshotContext<T> extends AsyncRpcContext<T> {
         final SnapshotInfo snapshot;
@@ -244,7 +253,7 @@ public class SnapshotServiceImpl implements SnapshotService {
         try {
             snapObj.processEvent(Snapshot.Event.BackupToSecondary);
 
-            DataStore imageStore = this.findSnapshotImageStore(snapshot);
+            DataStore imageStore = findSnapshotImageStore(snapshot);
             if (imageStore == null) {
                 throw new CloudRuntimeException("can not find an image stores");
             }
@@ -255,7 +264,7 @@ public class SnapshotServiceImpl implements SnapshotService {
             CopySnapshotContext<CommandResult> context = new CopySnapshotContext<CommandResult>(null, snapshot, snapshotOnImageStore, future);
             AsyncCallbackDispatcher<SnapshotServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
             caller.setCallback(caller.getTarget().copySnapshotAsyncCallback(null, null)).setContext(context);
-            this.motionSrv.copyAsync(snapshot, snapshotOnImageStore, caller);
+            motionSrv.copyAsync(snapshot, snapshotOnImageStore, caller);
         } catch (Exception e) {
             s_logger.debug("Failed to copy snapshot", e);
             result.setResult("Failed to copy snapshot:" + e.toString());
@@ -306,7 +315,7 @@ public class SnapshotServiceImpl implements SnapshotService {
             CopyCmdAnswer answer = (CopyCmdAnswer)result.getAnswer();
             destSnapshot.processEvent(Event.OperationSuccessed, result.getAnswer());
             srcSnapshot.processEvent(Snapshot.Event.OperationSucceeded);
-            snapResult = new SnapshotResult(this.snapshotfactory.getSnapshot(destSnapshot.getId(), destSnapshot.getDataStore()), answer);
+            snapResult = new SnapshotResult(_snapshotFactory.getSnapshot(destSnapshot.getId(), destSnapshot.getDataStore()), answer);
             future.complete(snapResult);
         } catch (Exception e) {
             s_logger.debug("Failed to update snapshot state", e);
@@ -391,7 +400,7 @@ public class SnapshotServiceImpl implements SnapshotService {
 
     @Override
     public boolean revertSnapshot(Long snapshotId) {
-        SnapshotInfo snapshot = snapshotfactory.getSnapshot(snapshotId, DataStoreRole.Primary);
+        SnapshotInfo snapshot = _snapshotFactory.getSnapshot(snapshotId, DataStoreRole.Primary);
         PrimaryDataStore store = (PrimaryDataStore)snapshot.getDataStore();
 
         AsyncCallFuture<SnapshotResult> future = new AsyncCallFuture<SnapshotResult>();
@@ -417,4 +426,92 @@ public class SnapshotServiceImpl implements SnapshotService {
         return false;
     }
 
+    // This routine is used to push snapshots currently on cache store, but not in region store to region store.
+    // used in migrating existing NFS secondary storage to S3. We chose to push all volume related snapshots to handle delta snapshots smoothly.
+    @Override
+    public void syncVolumeSnapshotsToRegionStore(long volumeId, DataStore store) {
+        if (dataStoreMgr.isRegionStore(store)) {
+            // list all backed up snapshots for the given volume
+            List<SnapshotVO> snapshots = _snapshotDao.listByStatus(volumeId, Snapshot.State.BackedUp);
+            if (snapshots != null) {
+                for (SnapshotVO snapshot : snapshots) {
+                    syncSnapshotToRegionStore(snapshot.getId(), store);
+                }
+            }
+        }
+    }
+
+    // push one individual snapshots currently on cache store to region store if it is not there already
+    private void syncSnapshotToRegionStore(long snapshotId, DataStore store) {
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("sync snapshot " + snapshotId + " from cache to object store...");
+        }
+        // if snapshot is already on region wide object store, check if it is really downloaded there (by checking install_path). Sync snapshot to region
+        // wide store if it is not there physically.
+        SnapshotInfo snapOnStore = _snapshotFactory.getSnapshot(snapshotId, store);
+        if (snapOnStore == null) {
+            throw new CloudRuntimeException("Cannot find an entry in snapshot_store_ref for snapshot " + snapshotId + " on region store: " + store.getName());
+        }
+        if (snapOnStore.getPath() == null || snapOnStore.getPath().length() == 0) {
+            // snapshot is not on region store yet, sync to region store
+            SnapshotInfo srcSnapshot = _snapshotFactory.getReadySnapshotOnCache(snapshotId);
+            if (srcSnapshot == null) {
+                throw new CloudRuntimeException("Cannot find snapshot " + snapshotId + "  on cache store");
+            }
+            AsyncCallFuture<SnapshotResult> future = syncToRegionStoreAsync(srcSnapshot, store);
+            try {
+                SnapshotResult result = future.get();
+                if (result.isFailed()) {
+                    throw new CloudRuntimeException("sync snapshot from cache to region wide store failed for image store " + store.getName() + ":"
+                            + result.getResult());
+                }
+                _cacheMgr.releaseCacheObject(srcSnapshot); // reduce reference count for template on cache, so it can recycled by schedule
+            } catch (Exception ex) {
+                throw new CloudRuntimeException("sync snapshot from cache to region wide store failed for image store " + store.getName());
+            }
+        }
+
+    }
+
+    private AsyncCallFuture<SnapshotResult> syncToRegionStoreAsync(SnapshotInfo snapshot, DataStore store) {
+        AsyncCallFuture<SnapshotResult> future = new AsyncCallFuture<SnapshotResult>();
+        // no need to create entry on snapshot_store_ref here, since entries are already created when updateCloudToUseObjectStore is invoked.
+        // But we need to set default install path so that sync can be done in the right s3 path
+        SnapshotInfo snapshotOnStore = _snapshotFactory.getSnapshot(snapshot, store);
+        String installPath = TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/"
+                + snapshot.getAccountId() + "/" + snapshot.getVolumeId();
+        ((SnapshotObject)snapshotOnStore).setPath(installPath);
+        CopySnapshotContext<CommandResult> context = new CopySnapshotContext<CommandResult>(null, snapshot,
+                snapshotOnStore, future);
+        AsyncCallbackDispatcher<SnapshotServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher
+                .create(this);
+        caller.setCallback(caller.getTarget().syncSnapshotCallBack(null, null)).setContext(context);
+        motionSrv.copyAsync(snapshot, snapshotOnStore, caller);
+        return future;
+    }
+
+    protected Void syncSnapshotCallBack(AsyncCallbackDispatcher<SnapshotServiceImpl, CopyCommandResult> callback,
+            CopySnapshotContext<CommandResult> context) {
+        CopyCommandResult result = callback.getResult();
+        SnapshotInfo destSnapshot = context.destSnapshot;
+        SnapshotResult res = new SnapshotResult(destSnapshot, null);
+
+        AsyncCallFuture<SnapshotResult> future = context.future;
+        try {
+            if (result.isFailed()) {
+                res.setResult(result.getResult());
+                // no change to existing snapshot_store_ref, will try to re-sync later if other call triggers this sync operation
+            } else {
+                // this will update install path properly, next time it will not sync anymore.
+                destSnapshot.processEvent(Event.OperationSuccessed, result.getAnswer());
+            }
+            future.complete(res);
+        } catch (Exception e) {
+            s_logger.debug("Failed to process sync snapshot callback", e);
+            res.setResult(e.toString());
+            future.complete(res);
+        }
+
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
index b83fb4e..51c04bf 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
@@ -27,6 +27,9 @@ import java.util.List;
 
 import javax.inject.Inject;
 
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -38,8 +41,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.storage.LocalHostEndpoint;
 import org.apache.cloudstack.storage.RemoteHostEndPoint;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
 
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
@@ -211,7 +212,7 @@ public class DefaultEndPointSelector implements EndPointSelector {
 
     @Override
     public EndPoint select(DataObject srcData, DataObject destData, StorageAction action) {
-        if (action == StorageAction.BACKUPSNAPSHOT) {
+        if (action == StorageAction.BACKUPSNAPSHOT && srcData.getDataStore().getRole() == DataStoreRole.Primary) {
             SnapshotInfo srcSnapshot = (SnapshotInfo)srcData;
             if (srcSnapshot.getHypervisorType() == Hypervisor.HypervisorType.KVM) {
                 VolumeInfo volumeInfo = srcSnapshot.getBaseVolume();

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/3223028f/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
index 519d3db..28d6598 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
@@ -103,6 +103,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
         storeSnapshotSearch.and("snapshot_id", storeSnapshotSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ);
         storeSnapshotSearch.and("store_id", storeSnapshotSearch.entity().getDataStoreId(), SearchCriteria.Op.EQ);
         storeSnapshotSearch.and("store_role", storeSnapshotSearch.entity().getRole(), SearchCriteria.Op.EQ);
+        storeSnapshotSearch.and("state", storeSnapshotSearch.entity().getState(), SearchCriteria.Op.EQ);
         storeSnapshotSearch.done();
 
         snapshotIdSearch = createSearchBuilder();
@@ -335,6 +336,15 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
     }
 
     @Override
+    public SnapshotDataStoreVO findReadyOnCache(long snapshotId) {
+        SearchCriteria<SnapshotDataStoreVO> sc = storeSnapshotSearch.create();
+        sc.setParameters("snapshot_id", snapshotId);
+        sc.setParameters("store_role", DataStoreRole.ImageCache);
+        sc.setParameters("state", ObjectInDataStoreStateMachine.State.Ready);
+        return findOneIncludingRemovedBy(sc);
+    }
+
+    @Override
     public List<SnapshotDataStoreVO> listOnCache(long snapshotId) {
         SearchCriteria<SnapshotDataStoreVO> sc = storeSnapshotSearch.create();
         sc.setParameters("snapshot_id", snapshotId);