You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ni...@apache.org on 2013/07/28 14:12:18 UTC
git commit: updated refs/heads/4.2 to faeddc1
Updated Branches:
refs/heads/4.2 f83187585 -> faeddc15f
CLOUDSTACK-3659
Implement the download url expiration functionality for volumes
Signed off by : nitin mehta<ni...@citrix.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/faeddc15
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/faeddc15
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/faeddc15
Branch: refs/heads/4.2
Commit: faeddc15ff33246ee8d59ab188743d414701e816
Parents: f831875
Author: Nitin Mehta <ni...@citrix.com>
Authored: Sun Jul 28 17:35:10 2013 +0530
Committer: Nitin Mehta <ni...@citrix.com>
Committed: Sun Jul 28 17:40:26 2013 +0530
----------------------------------------------------------------------
.../datastore/db/VolumeDataStoreDao.java | 2 +
.../image/datastore/ImageStoreEntity.java | 2 +
.../storage/image/store/ImageStoreImpl.java | 4 +
.../storage/image/BaseImageStoreDriverImpl.java | 6 ++
.../storage/image/ImageStoreDriver.java | 4 +
.../image/db/VolumeDataStoreDaoImpl.java | 14 ++++
.../driver/CloudStackImageStoreDriverImpl.java | 21 +++++
.../com/cloud/storage/StorageManagerImpl.java | 82 +++++++++++++++-----
utils/src/com/cloud/utils/DateUtil.java | 11 +++
9 files changed, 127 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
index 698465f..cdc2999 100644
--- a/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
+++ b/engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
@@ -40,4 +40,6 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
VolumeDataStoreVO findByStoreVolume(long storeId, long volumeId, boolean lock);
List<VolumeDataStoreVO> listDestroyed(long storeId);
+
+ List<VolumeDataStoreVO> listVolumeDownloadUrls();
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
index 43a0f75..cdea293 100644
--- a/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
+++ b/engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
@@ -43,4 +43,6 @@ public interface ImageStoreEntity extends DataStore, ImageStore {
String getMountPoint(); // get the mount point on ssvm.
String createEntityExtractUrl(String installPath, ImageFormat format, DataObject dataObject); // get the entity download URL
+
+ void deleteExtractUrl(String installPath, String url, ImageFormat format);
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
index 855d8cb..00ac277 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
@@ -194,5 +194,9 @@ public class ImageStoreImpl implements ImageStoreEntity {
return driver.createEntityExtractUrl(this, installPath, format, dataObject);
}
+ @Override
+ public void deleteExtractUrl(String installPath, String url, ImageFormat format) {
+ driver.deleteEntityExtractUrl(this, installPath, url, format);
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index e2fc8b7..17b10f1 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -24,6 +24,7 @@ import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataTO;
import com.cloud.configuration.dao.ConfigurationDao;
+import com.cloud.storage.Storage;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
@@ -246,4 +247,9 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
@Override
public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback) {
}
+
+ @Override
+ public void deleteEntityExtractUrl(DataStore store, String installPath, String url, Storage.ImageFormat format){
+ }
+
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java b/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
index fa7ea37..2ca248f 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
@@ -25,5 +25,9 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
import com.cloud.storage.Storage.ImageFormat;
public interface ImageStoreDriver extends DataStoreDriver {
+
String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject);
+
+ void deleteEntityExtractUrl(DataStore store, String installPath, String url, ImageFormat format);
+
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
index 04f8b70..9771909 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
@@ -45,6 +45,7 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
private SearchBuilder<VolumeDataStoreVO> storeSearch;
private SearchBuilder<VolumeDataStoreVO> cacheSearch;
private SearchBuilder<VolumeDataStoreVO> storeVolumeSearch;
+ private SearchBuilder<VolumeDataStoreVO> downloadVolumeSearch;
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
@@ -77,6 +78,12 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ);
updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ);
updateStateSearch.done();
+
+ downloadVolumeSearch = createSearchBuilder();
+ downloadVolumeSearch.and("download_url", downloadVolumeSearch.entity().getExtractUrl(), Op.NNULL);
+ downloadVolumeSearch.and("destroyed", downloadVolumeSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
+ downloadVolumeSearch.done();
+
return true;
}
@@ -186,4 +193,11 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
sc.setParameters("destroyed", true);
return listIncludingRemovedBy(sc);
}
+
+ @Override
+ public List<VolumeDataStoreVO> listVolumeDownloadUrls() {
+ SearchCriteria<VolumeDataStoreVO> sc = downloadVolumeSearch.create();
+ sc.setParameters("destroyed", false);
+ return listBy(sc);
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
index 6001c54..15d294c 100644
--- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
+++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
@@ -23,6 +23,10 @@ import java.util.UUID;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
+
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Upload;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
@@ -76,6 +80,23 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl {
return generateCopyUrl(ep.getPublicAddr(), uuid);
}
+ @Override
+ public void deleteEntityExtractUrl(DataStore store, String installPath, String downloadUrl, ImageFormat format) {
+ // find an endpoint to send command
+ EndPoint ep = _epSelector.select(store);
+ // Create Symlink at ssvm
+ //CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity) store).getMountPoint(), installPath, uuid);
+ DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(installPath, Upload.Type.VOLUME, downloadUrl, ((ImageStoreEntity) store).getMountPoint());
+
+ Answer ans = ep.sendMessage(cmd);
+ if (ans == null || !ans.getResult()) {
+ String errorString = "Unable to delete the url " + downloadUrl + " for path " + installPath + " on ssvm, " + ans.getDetails();
+ s_logger.error(errorString);
+ throw new CloudRuntimeException(errorString);
+ }
+
+ }
+
private String generateCopyUrl(String ipAddress, String uuid){
String hostname = ipAddress;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/server/src/com/cloud/storage/StorageManagerImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java
index 235c694..11f6540 100755
--- a/server/src/com/cloud/storage/StorageManagerImpl.java
+++ b/server/src/com/cloud/storage/StorageManagerImpl.java
@@ -41,6 +41,8 @@ import javax.ejb.Local;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.utils.DateUtil;
import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
@@ -49,26 +51,8 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingSto
import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
-import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
-import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
-import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
-import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider;
-import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
-import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
-import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
-import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
+import org.apache.cloudstack.engine.subsystem.api.storage.*;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
-import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
@@ -81,6 +65,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
+import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@@ -257,6 +242,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
DataStoreManager _dataStoreMgr;
@Inject
DataStoreProviderManager _dataStoreProviderMgr;
+
@Inject
private TemplateService _imageSrv;
@Inject
@@ -291,6 +277,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
boolean _storageCleanupEnabled;
boolean _templateCleanupEnabled = true;
int _storageCleanupInterval;
+ int _downloadUrlCleanupInterval;
+ int _downloadUrlExpirationInterval;
private int _createVolumeFromSnapshotWait;
private int _copyvolumewait;
int _storagePoolAcquisitionWaitSeconds = 1800; // 30 minutes
@@ -510,6 +498,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
int wrks = NumbersUtil.parseInt(workers, 10);
_executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("StorageManager-Scavenger"));
+ String cleanupInterval = configs.get("extract.url.cleanup.interval");
+ _downloadUrlCleanupInterval = NumbersUtil.parseInt(cleanupInterval, 7200);
+
+ String urlExpirationInterval = configs.get("extract.url.expiration.interval");
+ _downloadUrlExpirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400);
+
_agentMgr.registerForHostEvents(ComponentContext.inject(LocalStoragePoolListener.class), true, false, false);
String maxVolumeSizeInGbString = _configDao.getValue("storage.max.volume.size");
@@ -571,6 +565,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
} else {
s_logger.debug("Storage cleanup is not enabled, so the storage cleanup thread is not being scheduled.");
}
+
+ _executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
return true;
}
@@ -1097,6 +1093,35 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
+
+ public void cleanupDownloadUrls(){
+ // find volumesOnImageStoreList with download url
+ List<VolumeDataStoreVO> volumesOnImageStoreList = _volumeStoreDao.listVolumeDownloadUrls();
+
+ for(VolumeDataStoreVO volumeOnImageStore : volumesOnImageStoreList){
+
+ try {
+ long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(), volumeOnImageStore.getUpdated());
+ if(downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval){ // URL hasnt expired yet
+ continue;
+ }
+
+ s_logger.debug("Removing download url " + volumeOnImageStore.getExtractUrl() + " for volume id " + volumeOnImageStore.getVolumeId());
+
+ // Remove it from image store
+ ImageStoreEntity secStore = (ImageStoreEntity) _dataStoreMgr.getDataStore(volumeOnImageStore.getDataStoreId(), DataStoreRole.Image);
+ secStore.deleteExtractUrl(volumeOnImageStore.getInstallPath(), volumeOnImageStore.getExtractUrl(), ImageFormat.VHD);
+
+ // Now remove it from DB.
+ volumeOnImageStore.setExtractUrl(null);
+ _volumeStoreDao.update(volumeOnImageStore.getId(), volumeOnImageStore);
+ }catch(Throwable th){
+ s_logger.warn("caught exception while deleting download url " +volumeOnImageStore.getExtractUrl(), th);
+ }
+ }
+
+ }
+
@Override
@DB
public void cleanupSecondaryStorage(boolean recurring) {
@@ -1262,6 +1287,25 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
+
+ protected class DownloadURLGarbageCollector implements Runnable {
+
+ public DownloadURLGarbageCollector() {
+ }
+
+ @Override
+ public void run() {
+ try {
+ s_logger.trace("Download URL Garbage Collection Thread is running.");
+
+ cleanupDownloadUrls();
+
+ } catch (Exception e) {
+ s_logger.error("Caught the following Exception", e);
+ }
+ }
+ }
+
@Override
public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
// TODO Auto-generated method stub
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/faeddc15/utils/src/com/cloud/utils/DateUtil.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/DateUtil.java b/utils/src/com/cloud/utils/DateUtil.java
index 1854e15..af89046 100644
--- a/utils/src/com/cloud/utils/DateUtil.java
+++ b/utils/src/com/cloud/utils/DateUtil.java
@@ -228,6 +228,17 @@ public class DateUtil {
return scheduleTime.getTime();
}
+
+ public static long getTimeDifference(Date date1, Date date2){
+
+ Calendar dateCalendar1 = Calendar.getInstance();
+ dateCalendar1.setTime(date1);
+ Calendar dateCalendar2 = Calendar.getInstance();
+ dateCalendar2.setTime(date2);
+
+ return (dateCalendar1.getTimeInMillis() - dateCalendar2.getTimeInMillis() )/1000;
+
+ }
// test only
public static void main(String[] args) {