You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by mc...@apache.org on 2013/04/15 23:45:02 UTC
git commit: updated refs/heads/object_store to 5933375
Updated Branches:
refs/heads/object_store 2dd8e2cb9 -> 593337565
Add S3 code to handle DownloadCommand for registerTemplate.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/59333756
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/59333756
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/59333756
Branch: refs/heads/object_store
Commit: 593337565e75f114075053dc06f51165bb862508
Parents: 2dd8e2c
Author: Min Chen <mi...@citrix.com>
Authored: Mon Apr 15 14:44:39 2013 -0700
Committer: Min Chen <mi...@citrix.com>
Committed: Mon Apr 15 14:44:39 2013 -0700
----------------------------------------------------------------------
.../cloud/agent/api/storage/DownloadCommand.java | 29 ++++-
api/src/com/cloud/agent/api/to/NfsTO.java | 60 +++++++++
.../resource/NfsSecondaryStorageResource.java | 97 +++++++++++----
.../storage/template/DownloadManagerImpl.java | 40 +++++--
.../com/cloud/agent/transport/RequestTest.java | 18 ++--
.../storage/motion/AncientDataMotionStrategy.java | 41 +++---
.../storage/image/store/ImageStoreImpl.java | 2 +-
.../storage/datastore/PrimaryDataStoreImpl.java | 14 +-
.../driver/CloudStackImageStoreDriverImpl.java | 6 +-
.../storage/download/DownloadMonitorImpl.java | 2 +-
utils/src/com/cloud/utils/S3Utils.java | 21 +++
utils/src/com/cloud/utils/UriUtils.java | 81 ++++++++++++
12 files changed, 336 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/api/src/com/cloud/agent/api/storage/DownloadCommand.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/storage/DownloadCommand.java b/api/src/com/cloud/agent/api/storage/DownloadCommand.java
index c6ffe45..77e7ae3 100644
--- a/api/src/com/cloud/agent/api/storage/DownloadCommand.java
+++ b/api/src/com/cloud/agent/api/storage/DownloadCommand.java
@@ -20,6 +20,7 @@ import java.net.URI;
import org.apache.cloudstack.api.InternalIdentity;
+import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Volume;
import com.cloud.template.VirtualMachineTemplate;
@@ -105,6 +106,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
private Long maxDownloadSizeInBytes = null;
private long id;
private ResourceType resourceType = ResourceType.TEMPLATE;
+ private DataStoreTO _store;
+ private Long resourceId;
protected DownloadCommand() {
}
@@ -122,14 +125,16 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
this.resourceType = that.resourceType;
}
- public DownloadCommand(String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes) {
+ public DownloadCommand(DataStoreTO store, String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes) {
super(template.getUniqueName(), template.getUrl(), template.getFormat(), template.getAccountId());
+ this._store = store;
this.hvm = template.isRequiresHvm();
this.checksum = template.getChecksum();
this.id = template.getId();
this.description = template.getDisplayText();
this.setSecUrl(secUrl);
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+ this.resourceId = template.getId();
}
public DownloadCommand(String secUrl, Volume volume, Long maxDownloadSizeInBytes, String checkSum, String url, ImageFormat format) {
@@ -216,4 +221,26 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
public void setResourceType(ResourceType resourceType) {
this.resourceType = resourceType;
}
+
+
+ public DataStoreTO getDataStore() {
+ return _store;
+ }
+
+
+ public void setDataStore(DataStoreTO _store) {
+ this._store = _store;
+ }
+
+
+ public Long getResourceId() {
+ return resourceId;
+ }
+
+
+ public void setResourceId(Long resourceId) {
+ this.resourceId = resourceId;
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/api/src/com/cloud/agent/api/to/NfsTO.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/to/NfsTO.java b/api/src/com/cloud/agent/api/to/NfsTO.java
new file mode 100644
index 0000000..5490fd1
--- /dev/null
+++ b/api/src/com/cloud/agent/api/to/NfsTO.java
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import com.cloud.storage.DataStoreRole;
+
+public final class NfsTO implements DataStoreTO {
+
+ private String _url;
+ private DataStoreRole _role;
+
+ public NfsTO() {
+
+ super();
+
+ }
+
+ public NfsTO(String url, DataStoreRole role) {
+
+ super();
+
+ this._url = url;
+ this._role = role;
+
+ }
+
+ public String getUrl() {
+ return _url;
+ }
+
+ public void setUrl(String _url) {
+ this._url = _url;
+ }
+
+ @Override
+ public DataStoreRole getRole() {
+ return _role;
+ }
+
+ public void setRole(DataStoreRole _role) {
+ this._role = _role;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
index a8788c6..776ce29 100755
--- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
+++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
@@ -34,7 +34,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
import java.net.InetAddress;
+import java.net.MalformedURLException;
import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@@ -90,6 +93,8 @@ import com.cloud.agent.api.storage.ListVolumeAnswer;
import com.cloud.agent.api.storage.ListVolumeCommand;
import com.cloud.agent.api.storage.UploadCommand;
import com.cloud.agent.api.storage.ssCommand;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.exception.InternalErrorException;
@@ -106,6 +111,7 @@ import com.cloud.storage.template.UploadManager;
import com.cloud.storage.template.UploadManagerImpl;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.S3Utils;
+import com.cloud.utils.UriUtils;
import com.cloud.utils.S3Utils.FileNamingStrategy;
import com.cloud.utils.S3Utils.ObjectNamingStrategy;
import com.cloud.utils.component.ComponentContext;
@@ -126,7 +132,7 @@ SecondaryStorageResource {
int _timeout;
- String _instance;
+ String _instance;
String _dc;
String _pod;
String _guid;
@@ -162,8 +168,8 @@ SecondaryStorageResource {
if (cmd instanceof DownloadProgressCommand) {
return _dlMgr.handleDownloadCommand(this, (DownloadProgressCommand)cmd);
} else if (cmd instanceof DownloadCommand) {
- return _dlMgr.handleDownloadCommand(this, (DownloadCommand)cmd);
- } else if (cmd instanceof UploadCommand) {
+ return execute((DownloadCommand)cmd);
+ } else if (cmd instanceof UploadCommand) {
return _upldMgr.handleUploadCommand(this, (UploadCommand)cmd);
} else if (cmd instanceof CreateEntityDownloadURLCommand){
return _upldMgr.handleCreateEntityURLCommand((CreateEntityDownloadURLCommand)cmd);
@@ -325,6 +331,51 @@ SecondaryStorageResource {
}
}
+ private Answer execute(DownloadCommand cmd){
+ DataStoreTO dstore = cmd.getDataStore();
+ if ( dstore instanceof NfsTO ){
+ return _dlMgr.handleDownloadCommand(this, cmd);
+ }
+ else if ( dstore instanceof S3TO ){
+ //TODO: how to handle download progress for S3
+ S3TO s3 = (S3TO)cmd.getDataStore();
+ String url = cmd.getUrl();
+ String user = null;
+ String password = null;
+ if (cmd.getAuth() != null) {
+ user = cmd.getAuth().getUserName();
+ password = new String(cmd.getAuth().getPassword());
+ }
+ // get input stream from the given url
+ InputStream in = UriUtils.getInputStreamFromUrl(url, user, password);
+ URI uri;
+ URL urlObj;
+ try {
+ uri = new URI(url);
+ urlObj = new URL(url);
+ } catch (URISyntaxException e) {
+ throw new CloudRuntimeException("URI is incorrect: " + url);
+ } catch (MalformedURLException e) {
+ throw new CloudRuntimeException("URL is incorrect: " + url);
+ }
+
+ final String bucket = s3.getBucketName();
+ String key = join(asList(determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId()), urlObj.getFile()), S3Utils.SEPARATOR);
+ S3Utils.putObject(s3, in, bucket, key);
+ return new Answer(cmd, true, format("Uploaded the contents of input stream from %1$s for template id %2$s to S3 bucket %3$s", url,
+ cmd.getResourceId(), bucket));
+ }
+ else if ( dstore instanceof SwiftTO ){
+ //TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand) here, but we need to handle
+ // source is url, most likely we need to modify our existing swiftUpload python script.
+ return new Answer(cmd, false, "Swift is not currently support DownloadCommand");
+ }
+ else{
+ return new Answer(cmd, false, "Unsupport image data store: " + dstore);
+ }
+
+ }
+
private Answer execute(uploadTemplateToSwiftFromSecondaryStorageCommand cmd) {
SwiftTO swift = cmd.getSwift();
String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
@@ -834,33 +885,33 @@ SecondaryStorageResource {
String absoluteTemplatePath = parent + relativeTemplatePath;
MessageDigest digest;
String checksum = null;
- File f = new File(absoluteTemplatePath);
+ File f = new File(absoluteTemplatePath);
InputStream is = null;
byte[] buffer = new byte[8192];
int read = 0;
if(s_logger.isDebugEnabled()){
- s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath );
+ s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath );
}
try {
- digest = MessageDigest.getInstance("MD5");
- is = new FileInputStream(f);
+ digest = MessageDigest.getInstance("MD5");
+ is = new FileInputStream(f);
while( (read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
- }
+ }
byte[] md5sum = digest.digest();
BigInteger bigInt = new BigInteger(1, md5sum);
checksum = bigInt.toString(16);
if(s_logger.isDebugEnabled()){
- s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+ " - " +checksum );
+ s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+ " - " +checksum );
}
}catch(IOException e) {
String logMsg = "Unable to process file for MD5 - " + absoluteTemplatePath;
s_logger.error(logMsg);
- return new Answer(cmd, false, checksum);
- }catch (NoSuchAlgorithmException e) {
+ return new Answer(cmd, false, checksum);
+ }catch (NoSuchAlgorithmException e) {
return new Answer(cmd, false, checksum);
}
finally {
@@ -869,10 +920,10 @@ SecondaryStorageResource {
is.close();
} catch (IOException e) {
if(s_logger.isDebugEnabled()){
- s_logger.debug("Could not close the file " +absoluteTemplatePath);
+ s_logger.debug("Could not close the file " +absoluteTemplatePath);
}
- return new Answer(cmd, false, checksum);
- }
+ return new Answer(cmd, false, checksum);
+ }
}
return new Answer(cmd, true, checksum);
@@ -1141,7 +1192,7 @@ SecondaryStorageResource {
if (nfsIps.contains(cidr)) {
/*
* if the internal download ip is the same with secondary storage ip, adding internal sites will flush
- * ip route to nfs through storage ip.
+ * ip route to nfs through storage ip.
*/
continue;
}
@@ -1243,7 +1294,7 @@ SecondaryStorageResource {
for (PortConfig pCfg:cmd.getPortConfigs()){
if (pCfg.isAdd()) {
- ipList.add(pCfg.getSourceIp());
+ ipList.add(pCfg.getSourceIp());
}
}
boolean success = true;
@@ -1401,7 +1452,7 @@ SecondaryStorageResource {
String nfsPath = nfsHostIp + ":" + uri.getPath();
String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString();
String root = _parent + "/" + dir;
- mount(root, nfsPath);
+ mount(root, nfsPath);
return root;
} catch (Exception e) {
String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString();
@@ -1470,7 +1521,7 @@ SecondaryStorageResource {
String eth2ip = (String) params.get("eth2ip");
if (eth2ip != null) {
params.put("public.network.device", "eth2");
- }
+ }
_publicIp = (String) params.get("eth2ip");
_hostname = (String) params.get("name");
@@ -1677,7 +1728,7 @@ SecondaryStorageResource {
command.add(String.valueOf(isAppend));
for (String ip : ipList){
command.add(ip);
- }
+ }
String result = command.execute();
if (result != null) {
@@ -1693,7 +1744,7 @@ SecondaryStorageResource {
s_logger.debug("create mount point: " + root);
} else {
s_logger.debug("Unable to create mount point: " + root);
- return null;
+ return null;
}
}
@@ -1813,13 +1864,13 @@ SecondaryStorageResource {
@Override
public void setName(String name) {
// TODO Auto-generated method stub
-
+
}
@Override
public void setConfigParams(Map<String, Object> params) {
// TODO Auto-generated method stub
-
+
}
@Override
@@ -1837,6 +1888,6 @@ SecondaryStorageResource {
@Override
public void setRunLevel(int level) {
// TODO Auto-generated method stub
-
+
}
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/core/src/com/cloud/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java
index db264c1..7eb6841 100755
--- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java
+++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java
@@ -16,9 +16,15 @@
// under the License.
package com.cloud.storage.template;
+import static com.cloud.utils.S3Utils.putDirectory;
+import static com.cloud.utils.StringUtils.join;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigInteger;
@@ -41,14 +47,21 @@ import java.util.concurrent.Executors;
import javax.ejb.Local;
import javax.naming.ConfigurationException;
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.log4j.Logger;
+import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.DownloadCommand;
import com.cloud.agent.api.storage.DownloadCommand.Proxy;
import com.cloud.agent.api.storage.DownloadCommand.ResourceType;
import com.cloud.agent.api.storage.DownloadProgressCommand;
import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType;
+import com.cloud.agent.api.to.S3TO;
import com.cloud.exception.InternalErrorException;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageLayer;
@@ -59,6 +72,9 @@ import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
import com.cloud.storage.template.TemplateDownloader.Status;
import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.S3Utils;
+import com.cloud.utils.UriUtils;
+import com.cloud.utils.S3Utils.ObjectNamingStrategy;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
@@ -224,7 +240,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
/**
* Get notified of change of job status. Executed in context of downloader thread
- *
+ *
* @param jobId
* the id of the job
* @param status
@@ -279,21 +295,21 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
byte[] buffer = new byte[8192];
int read = 0;
MessageDigest digest;
- String checksum = null;
+ String checksum = null;
InputStream is = null;
try {
- digest = MessageDigest.getInstance("MD5");
- is = new FileInputStream(f);
+ digest = MessageDigest.getInstance("MD5");
+ is = new FileInputStream(f);
while( (read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
- }
+ }
byte[] md5sum = digest.digest();
BigInteger bigInt = new BigInteger(1, md5sum);
checksum = String.format("%032x",bigInt);
return checksum;
}catch(IOException e) {
return null;
- }catch (NoSuchAlgorithmException e) {
+ }catch (NoSuchAlgorithmException e) {
return null;
}
finally {
@@ -302,19 +318,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
is.close();
} catch (IOException e) {
return null;
- }
+ }
}
}
/**
* Post download activity (install and cleanup). Executed in context of downloader thread
- *
+ *
* @throws IOException
*/
private String postDownload(String jobId) {
DownloadJob dnld = jobs.get(jobId);
TemplateDownloader td = dnld.getTemplateDownloader();
- String resourcePath = null;
+ String resourcePath = null;
ResourceType resourceType = dnld.getResourceType();
// once template path is set, remove the parent dir so that the template is installed with a relative path
@@ -458,7 +474,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
return "Unable to create " + tmpDir;
}
// TO DO - define constant for volume properties.
- File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) :
+ File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) :
_storage.getFile(tmpDir + File.separator + "volume.properties");
if ( file.exists() ) {
file.delete();
@@ -583,6 +599,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
return convertStatus(getDownloadStatus(jobId));
}
+
@Override
public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) {
ResourceType resourceType = cmd.getResourceType();
@@ -774,7 +791,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
@Override
- public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {
+ public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {
Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>();
String volumeDir = rootDir + File.separator + _volumeDir;
@@ -1046,4 +1063,5 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
}
+
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/core/test/com/cloud/agent/transport/RequestTest.java
----------------------------------------------------------------------
diff --git a/core/test/com/cloud/agent/transport/RequestTest.java b/core/test/com/cloud/agent/transport/RequestTest.java
index 64c1e0b..048bd21 100644
--- a/core/test/com/cloud/agent/transport/RequestTest.java
+++ b/core/test/com/cloud/agent/transport/RequestTest.java
@@ -31,19 +31,21 @@ import com.cloud.agent.api.SecStorageFirewallCfgCommand;
import com.cloud.agent.api.UpdateHostPasswordCommand;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.DownloadCommand;
+import com.cloud.agent.api.to.NfsTO;
import com.cloud.exception.UnsupportedVersionException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.serializer.GsonHelper;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateVO;
/**
- *
- *
- *
- *
+ *
+ *
+ *
+ *
*/
public class RequestTest extends TestCase {
@@ -51,7 +53,7 @@ public class RequestTest extends TestCase {
public void testSerDeser() {
s_logger.info("Testing serializing and deserializing works as expected");
-
+
s_logger.info("UpdateHostPasswordCommand should have two parameters that doesn't show in logging");
UpdateHostPasswordCommand cmd1 = new UpdateHostPasswordCommand("abc", "def");
s_logger.info("SecStorageFirewallCfgCommand has a context map that shouldn't show up in debug level");
@@ -89,7 +91,7 @@ public class RequestTest extends TestCase {
logger.setLevel(level);
byte[] bytes = sreq.getBytes();
-
+
assert Request.getSequence(bytes) == 892403717;
assert Request.getManagementServerId(bytes) == 3;
assert Request.getAgentId(bytes) == 2;
@@ -130,7 +132,7 @@ public class RequestTest extends TestCase {
s_logger.info("Testing Download answer");
VMTemplateVO template = new VMTemplateVO(1, "templatename", ImageFormat.QCOW2, true, true, true, TemplateType.USER, "url", true, 32, 1, "chksum", "displayText", true, 30, true,
HypervisorType.KVM, null);
- DownloadCommand cmd = new DownloadCommand("secUrl", template, 30000000l);
+ DownloadCommand cmd = new DownloadCommand(new NfsTO("secUrl", DataStoreRole.Image), "secUrl", template, 30000000l);
Request req = new Request(1, 1, cmd, true);
req.logD("Debug for Download");
@@ -161,7 +163,7 @@ public class RequestTest extends TestCase {
}
}
}
-
+
public void testLogging() {
s_logger.info("Testing Logging");
GetHostStatsCommand cmd3 = new GetHostStatsCommand("hostguid", "hostname", 101);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
----------------------------------------------------------------------
diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
index efc2f99..e72bb54 100644
--- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
+++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
@@ -28,7 +28,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole;
+
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
@@ -65,6 +65,7 @@ import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.exception.StorageUnavailableException;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
+import com.cloud.storage.DataStoreRole;
import com.cloud.storage.DiskOfferingVO;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Storage.ImageFormat;
@@ -130,7 +131,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
VolumeManager volumeMgr;
@Inject
private SwiftManager _swiftMgr;
- @Inject
+ @Inject
private S3Manager _s3Mgr;
@Inject
StorageCacheManager cacheMgr;
@@ -314,12 +315,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
}
}
}
-
+
protected Answer cloneVolume(DataObject template, DataObject volume) {
VolumeInfo volInfo = (VolumeInfo)volume;
DiskOfferingVO offering = diskOfferingDao.findById(volInfo.getDiskOfferingId());
VMTemplateStoragePoolVO tmpltStoredOn = templatePoolDao.findByPoolTemplate(template.getDataStore().getId(), template.getId());
-
+
DiskProfile diskProfile = new DiskProfile(volInfo, offering,
null);
CreateCommand cmd = new CreateCommand(diskProfile,
@@ -334,7 +335,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
s_logger.debug("Failed to send to storage pool", e);
throw new CloudRuntimeException("Failed to send to storage pool", e);
}
-
+
if (answer.getResult()) {
VolumeVO vol = this.volDao.findById(volume.getId());
CreateAnswer createAnswer = (CreateAnswer) answer;
@@ -345,7 +346,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
vol.setPoolId(pool.getId());
vol.setPodId(pool.getPodId());
this.volDao.update(vol.getId(), vol);
-
+
} else {
if (tmpltStoredOn != null
&& (answer instanceof CreateAnswer)
@@ -354,15 +355,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
if (!templateMgr
.resetTemplateDownloadStateOnPool(tmpltStoredOn
.getId())) {
-
+
}
}
errMsg = answer.getDetails();
}
-
+
return answer;
}
-
+
protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData) {
VolumeInfo volume = (VolumeInfo)srcData;
VolumeInfo destVolume = (VolumeInfo)destData;
@@ -370,9 +371,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
.getDataCenterId());
StoragePool srcPool = (StoragePool)this.dataStoreMgr.getDataStore(volume
.getPoolId(), DataStoreRole.Primary);
-
+
StoragePool destPool = (StoragePool)this.dataStoreMgr.getDataStore(destVolume.getPoolId(), DataStoreRole.Primary);
-
+
String value = this.configDao.getValue(Config.CopyVolumeWait.toString());
int _copyvolumewait = NumbersUtil.parseInt(value,
Integer.parseInt(Config.CopyVolumeWait.getDefaultValue()));
@@ -394,7 +395,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
}
String secondaryStorageVolumePath = cvAnswer.getVolumePath();
-
+
cvCmd = new CopyVolumeCommand(volume.getId(),
secondaryStorageVolumePath, destPool,
secondaryStorageURL, false, _copyvolumewait);
@@ -409,7 +410,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
throw new CloudRuntimeException(
"Failed to copy the volume from secondary storage to the destination primary storage pool.");
}
-
+
VolumeVO destVol = this.volDao.findById(destVolume.getId());
destVol.setPath(cvAnswer.getVolumePath());
this.volDao.update(destVol.getId(), destVol);
@@ -437,7 +438,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
} else if (srcData.getType() == DataObjectType.VOLUME
&& destData.getType() == DataObjectType.TEMPLATE) {
answer = createTemplateFromVolume(srcData, destData);
- } else if (srcData.getType() == DataObjectType.TEMPLATE
+ } else if (srcData.getType() == DataObjectType.TEMPLATE
&& destData.getType() == DataObjectType.VOLUME) {
answer = cloneVolume(srcData, destData);
} else if (destData.getType() == DataObjectType.VOLUME
@@ -667,21 +668,21 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
return sendCommand(cmd, pool, template.getId(), zoneId,
secondaryStorageHost.getId());
}
-
+
private HostVO getSecHost(long volumeId, long dcId) {
Long id = snapshotDao.getSecHostId(volumeId);
- if ( id != null) {
+ if ( id != null) {
return hostDao.findById(id);
}
return this.templateMgr.getSecondaryStorageHost(dcId);
}
-
+
protected Answer copySnapshot(DataObject srcObject, DataObject destObject) {
SnapshotInfo srcSnapshot = (SnapshotInfo)srcObject;
VolumeInfo baseVolume = srcSnapshot.getBaseVolume();
Long dcId = baseVolume.getDataCenterId();
Long accountId = baseVolume.getAccountId();
-
+
HostVO secHost = getSecHost(baseVolume.getId(), baseVolume.getDataCenterId());
Long secHostId = secHost.getId();
String secondaryStoragePoolUrl = secHost.getStorageUrl();
@@ -696,12 +697,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
SwiftTO swift = _swiftMgr.getSwiftTO();
S3TO s3 = _s3Mgr.getS3TO();
-
+
long prevSnapshotId = srcSnapshot.getPrevSnapshotId();
if (prevSnapshotId > 0) {
prevSnapshot = snapshotDao.findByIdIncludingRemoved(prevSnapshotId);
if ( prevSnapshot.getBackupSnapshotId() != null && swift == null) {
- if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2")) {
+ if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2")) {
prevBackupUuid = prevSnapshot.getBackupSnapshotId();
prevSnapshotUuid = prevSnapshot.getPath();
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
index 4989b26..bc8d2e2 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
@@ -175,7 +175,7 @@ public class ImageStoreImpl implements ImageStoreEntity {
@Override
public DataStoreTO getTO() {
- return null;
+ return getDriver().getStoreTO(this);
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
index 0feefd0..8be1c8a 100644
--- a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
+++ b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java
@@ -83,10 +83,10 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
private VolumeDao volumeDao;
- public PrimaryDataStoreImpl() {
-
+ public PrimaryDataStoreImpl() {
+
}
-
+
public void configure(StoragePoolVO pdsv,
PrimaryDataStoreDriver driver, DataStoreProvider provider) {
this.pdsv = pdsv;
@@ -245,9 +245,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
}
}
}
-
+
}
-
+
return objectInStoreMgr.get(obj, this);
}
@@ -341,9 +341,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
public String getStorageProviderName() {
return this.pdsv.getStorageProviderName();
}
-
+
@Override
public DataStoreTO getTO() {
- return null;
+ return getDriver().getStoreTO(this);
}
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
index af6a3b6..f902385 100644
--- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
+++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
@@ -46,6 +46,7 @@ import com.cloud.agent.api.Answer;
import com.cloud.agent.api.DeleteSnapshotBackupCommand;
import com.cloud.agent.api.storage.DeleteVolumeCommand;
import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.host.HostVO;
@@ -106,10 +107,9 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
@Override
public DataStoreTO getStoreTO(DataStore store) {
ImageStoreImpl nfsStore = (ImageStoreImpl)store;
- ImageStoreTO nfsTO = new ImageStoreTO();
- nfsTO.setProviderName("CloudStack");
+ NfsTO nfsTO = new NfsTO();
nfsTO.setRole(DataStoreRole.Image);
- nfsTO.setUri(nfsStore.getUri());
+ nfsTO.setUrl(nfsStore.getUri());
return nfsTO;
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
index 507d575..58477b4 100755
--- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
+++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java
@@ -366,7 +366,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
if(vmTemplateStore != null) {
start();
DownloadCommand dcmd =
- new DownloadCommand(secUrl, template, maxTemplateSizeInBytes);
+ new DownloadCommand(store.getTO(), secUrl, template, maxTemplateSizeInBytes);
dcmd.setProxy(getHttpProxy());
if (downloadJobExists) {
dcmd = new DownloadProgressCommand(dcmd, vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/utils/src/com/cloud/utils/S3Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java
index b7273a1..423ea71 100644
--- a/utils/src/com/cloud/utils/S3Utils.java
+++ b/utils/src/com/cloud/utils/S3Utils.java
@@ -138,6 +138,24 @@ public final class S3Utils {
}
+ public static void putObject(final ClientOptions clientOptions,
+ final InputStream sourceStream, final String bucketName, final String key) {
+
+ assert clientOptions != null;
+ assert sourceStream != null;
+ assert !isBlank(bucketName);
+ assert !isBlank(key);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(format("Sending stream as S3 object %1$s in "
+ + "bucket %2$s", key, bucketName));
+ }
+
+ acquireClient(clientOptions).putObject(bucketName, key, sourceStream, null);
+
+ }
+
+
@SuppressWarnings("unchecked")
public static File getFile(final ClientOptions clientOptions,
final String bucketName, final String key,
@@ -239,6 +257,7 @@ public final class S3Utils {
}
+
public static void putDirectory(final ClientOptions clientOptions,
final String bucketName, final File directory,
final FilenameFilter fileNameFilter,
@@ -284,6 +303,8 @@ public final class S3Utils {
}
+
+
public static void deleteObject(final ClientOptions clientOptions,
final String bucketName, final String key) {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/59333756/utils/src/com/cloud/utils/UriUtils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java
index 3bcee7a..ef1aba2 100644
--- a/utils/src/com/cloud/utils/UriUtils.java
+++ b/utils/src/com/cloud/utils/UriUtils.java
@@ -18,16 +18,34 @@ package com.cloud.utils;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.net.HttpURLConnection;
+import java.net.Inet6Address;
+import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
+import java.net.UnknownHostException;
import javax.net.ssl.HttpsURLConnection;
+import org.apache.commons.httpclient.Credentials;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.UsernamePasswordCredentials;
+import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.log4j.Logger;
+
import com.cloud.utils.exception.CloudRuntimeException;
public class UriUtils {
+
+ public static final Logger s_logger = Logger.getLogger(UriUtils.class.getName());
+
public static String formNfsUri(String host, String path) {
try {
URI uri = new URI("nfs", host, path, null);
@@ -111,4 +129,67 @@ public class UriUtils {
}
return remoteSize;
}
+
+ public static Pair<String, Integer> validateUrl(String url) throws IllegalArgumentException {
+ try {
+ URI uri = new URI(url);
+ if (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") ) {
+ throw new IllegalArgumentException("Unsupported scheme for url");
+ }
+ int port = uri.getPort();
+ if (!(port == 80 || port == 443 || port == -1)) {
+ throw new IllegalArgumentException("Only ports 80 and 443 are allowed");
+ }
+
+ if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) {
+ port = 443;
+ } else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) {
+ port = 80;
+ }
+
+ String host = uri.getHost();
+ try {
+ InetAddress hostAddr = InetAddress.getByName(host);
+ if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) {
+ throw new IllegalArgumentException("Illegal host specified in url");
+ }
+ if (hostAddr instanceof Inet6Address) {
+ throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
+ }
+ return new Pair<String, Integer>(host, port);
+ } catch (UnknownHostException uhe) {
+ throw new IllegalArgumentException("Unable to resolve " + host);
+ }
+ } catch (URISyntaxException use) {
+ throw new IllegalArgumentException("Invalid URL: " + url);
+ }
+ }
+
+ public static InputStream getInputStreamFromUrl(String url, String user, String password) {
+
+ try{
+ Pair<String, Integer> hostAndPort = validateUrl(url);
+ HttpClient httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
+ if ((user != null) && (password != null)) {
+ httpclient.getParams().setAuthenticationPreemptive(true);
+ Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
+ httpclient.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
+ s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
+ }
+ // Execute the method.
+ GetMethod method = new GetMethod(url);
+ int statusCode = httpclient.executeMethod(method);
+
+ if (statusCode != HttpStatus.SC_OK) {
+ s_logger.error("Failed to read from URL: " + url);
+ return null;
+ }
+
+ return method.getResponseBodyAsStream();
+ }
+ catch (Exception ex){
+ s_logger.error("Failed to read from URL: " + url);
+ return null;
+ }
+ }
}