You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ra...@apache.org on 2015/03/03 13:00:28 UTC
git commit: updated refs/heads/volume-upload to b16520b
Repository: cloudstack
Updated Branches:
refs/heads/volume-upload dd1a8da97 -> b16520bce
volume upload: added max size check for temaplte/volume post upload
used the existing configuration variables max.template.iso.size and
storage.max.volume.upload.size for templates and volumes respectively.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/b16520bc
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/b16520bc
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/b16520bc
Branch: refs/heads/volume-upload
Commit: b16520bcecf5066259f9ee44c653fbb60847c18b
Parents: dd1a8da
Author: Rajani Karuturi <ra...@gmail.com>
Authored: Tue Mar 3 17:22:40 2015 +0530
Committer: Rajani Karuturi <ra...@gmail.com>
Committed: Tue Mar 3 17:27:40 2015 +0530
----------------------------------------------------------------------
.../TemplateOrVolumePostUploadCommand.java | 10 +++++++
.../com/cloud/storage/VolumeApiServiceImpl.java | 2 ++
.../template/HypervisorTemplateAdapter.java | 3 ++
.../resource/HttpUploadServerHandler.java | 22 +++++++++++----
.../resource/NfsSecondaryStorageResource.java | 29 ++++++++++++++++----
.../storage/template/UploadEntity.java | 7 +++++
6 files changed, 63 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java
----------------------------------------------------------------------
diff --git a/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java
index 8872798..cc9df71 100644
--- a/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java
+++ b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java
@@ -45,6 +45,8 @@ public class TemplateOrVolumePostUploadCommand {
String remoteEndPoint;
+ String maxUploadSize;
+
public TemplateOrVolumePostUploadCommand(long entityId, String entityUUID, String absolutePath, String checksum, String type, String name, String imageFormat, String dataTo,
String dataToRole) {
this.entityId = entityId;
@@ -156,4 +158,12 @@ public class TemplateOrVolumePostUploadCommand {
public void setName(String name) {
this.name = name;
}
+
+ public String getMaxUploadSize() {
+ return maxUploadSize;
+ }
+
+ public void setMaxUploadSize(String maxUploadSize) {
+ this.maxUploadSize = maxUploadSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/server/src/com/cloud/storage/VolumeApiServiceImpl.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java
index 9840096..d2c1c69 100644
--- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java
+++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java
@@ -337,6 +337,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
vol.getName(), vol.getFormat().toString(), dataObject.getDataStore().getUri(),
dataObject.getDataStore().getRole().toString());
command.setLocalPath(volumeStore.getLocalDownloadPath());
+ //using the existing max upload size configuration
+ command.setMaxUploadSize(_configDao.getValue(Config.MaxUploadVolumeSize.key()));
Gson gson = new GsonBuilder().create();
String metadata = EncryptionUtil.encodeData(gson.toJson(command), key);
response.setMetadata(metadata);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/server/src/com/cloud/template/HypervisorTemplateAdapter.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java
index 6d81047..0cb48fc 100755
--- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java
+++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java
@@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionException;
import javax.ejb.Local;
import javax.inject.Inject;
+import com.cloud.configuration.Config;
import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -259,6 +260,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl.getChecksum(), tmpl
.getType().toString(), template.getName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), templateOnStore.getDataStore().getRole()
.toString());
+ //using the existing max template size configuration
+ payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key()));
payload.setRemoteEndPoint(ep.getPublicAddr());
payload.setRequiresHvm(template.requiresHvm());
payloads.add(payload);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
index 0c95707..7a17ef1 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
@@ -108,6 +108,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
String expires = null;
String metadata = null;
String hostname = null;
+ long contentLength = 0;
+
for (Entry<String, String> entry : request.headers()) {
switch (entry.getKey()) {
case HEADER_SIGNATURE:
@@ -122,12 +124,16 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
case HEADER_HOST:
hostname = entry.getValue();
break;
+ case HttpHeaders.Names.CONTENT_LENGTH:
+ contentLength = Long.valueOf(entry.getValue());
+ break;
}
}
logger.info("HEADER: signature=" + signature);
logger.info("HEADER: metadata=" + metadata);
logger.info("HEADER: expires=" + expires);
logger.info("HEADER: hostname=" + hostname);
+ logger.info("HEADER: Content-Length=" + contentLength);
QueryStringDecoder decoderQuery = new QueryStringDecoder(uri);
Map<String, List<String>> uriAttributes = decoderQuery.parameters();
uuid = uriAttributes.get("uuid").get(0);
@@ -136,9 +142,9 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
UploadEntity uploadEntity = null;
try {
// Validate the request here
- storageResource.validatePostUploadRequest(signature, metadata, expires, hostname, uuid);
+ storageResource.validatePostUploadRequest(signature, metadata, expires, hostname, contentLength, uuid);
//create an upload entity. This will fail if entity already exists.
- uploadEntity = storageResource.createUploadEntity(uuid, metadata);
+ uploadEntity = storageResource.createUploadEntity(uuid, metadata, contentLength);
} catch (InvalidParameterValueException ex) {
logger.error("post request validation failed", ex);
responseContent.append(ex.getMessage());
@@ -185,9 +191,15 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
return;
}
if (chunk instanceof LastHttpContent) {
- readFileUploadData();
- writeResponse(ctx.channel(), HttpResponseStatus.OK);
- reset();
+ try {
+ readFileUploadData();
+ writeResponse(ctx.channel(), HttpResponseStatus.OK);
+ reset();
+ } catch (InvalidParameterValueException e) {
+ logger.error("error during the file install.", e);
+ responseContent.append("\n").append(e.getMessage());
+ writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index f80572a..236498c 100755
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -2602,7 +2602,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return Script.findScript(scriptsDir, scriptname);
}
- public UploadEntity createUploadEntity(String uuid, String metadata) {
+ public UploadEntity createUploadEntity(String uuid, String metadata, long contentLength) {
TemplateOrVolumePostUploadCommand cmd = getTemplateOrVolumePostUploadCmd(metadata);
UploadEntity uploadEntity = null;
if(cmd == null ){
@@ -2610,10 +2610,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
throw new InvalidParameterValueException("unable to decode and deserialize metadata");
} else {
uuid = cmd.getEntityUUID();
- if(uploadEntityStateMap.containsKey(uuid)) {
+ if (uploadEntityStateMap.containsKey(uuid)) {
uploadEntity = uploadEntityStateMap.get(uuid);
throw new InvalidParameterValueException("The one time post url is already used and the upload is in " + uploadEntity.getUploadState() + " state.");
}
+ int maxSizeInGB = Integer.valueOf(cmd.getMaxUploadSize());
+ int contentLengthInGB = getSizeInGB(contentLength);
+ if (contentLengthInGB > maxSizeInGB) {
+ throw new InvalidParameterValueException("Maximum file upload size exceeded. Content Length received: " + contentLengthInGB + "GB. Maximum allowed size: " +
+ maxSizeInGB + "GB.");
+ }
try {
String absolutePath = cmd.getAbsolutePath();
uploadEntity = new UploadEntity(uuid, cmd.getEntityId(), UploadEntity.Status.IN_PROGRESS, cmd.getName(), absolutePath);
@@ -2627,6 +2633,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
uploadEntity.setInstallPathPrefix(installPathPrefix);
uploadEntity.setHvm(cmd.getRequiresHvm());
uploadEntity.setChksum(cmd.getChecksum());
+ uploadEntity.setMaxSizeInGB(maxSizeInGB);
// create a install dir
if (!_storage.exists(installPathPrefix)) {
_storage.mkdir(installPathPrefix);
@@ -2634,13 +2641,16 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
uploadEntityStateMap.put(uuid, uploadEntity);
} catch (Exception e) {
//upload entity will be null incase an exception occurs and the handler will not proceed.
- s_logger.debug("exception occured while creating upload entity " + e);
+ s_logger.error("exception occurred while creating upload entity ", e);
updateStateMapWithError(uuid, e.getMessage());
}
}
return uploadEntity;
}
+ private int getSizeInGB(long sizeInBytes) {
+ return (int)Math.ceil(sizeInBytes * 1.0d / (1024 * 1024 * 1024));
+ }
public String postUpload(String uuid, String filename) {
UploadEntity uploadEntity = uploadEntityStateMap.get(uuid);
@@ -2657,7 +2667,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
//}
//dnld.setCheckSum(checkSum);
- int imgSizeGigs = (int)Math.ceil(_storage.getSize(fileSavedTempLocation) * 1.0d / (1024 * 1024 * 1024));
+ int imgSizeGigs = getSizeInGB(_storage.getSize(fileSavedTempLocation));
+ int maxSize = uploadEntity.getMaxSizeInGB();
+ if(imgSizeGigs > maxSize) {
+ throw new InvalidParameterValueException("Maximum file upload size exceeded. Physical file size: "+imgSizeGigs+"GB. Maximum allowed size: "+maxSize+"GB.");
+ }
imgSizeGigs++; // add one just in case
long timeout = (long)imgSizeGigs * installTimeoutPerGig;
Script scr = new Script(getScriptLocation(resourceType), timeout, s_logger);
@@ -2771,13 +2785,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
uploadEntityStateMap.put(uuid, uploadEntity);
}
- public void validatePostUploadRequest(String signature, String metadata, String timeout, String hostname, String uuid) throws InvalidParameterValueException{
+ public void validatePostUploadRequest(String signature, String metadata, String timeout, String hostname,long contentLength, String uuid) throws InvalidParameterValueException{
// check none of the params are empty
if(StringUtils.isEmpty(signature) || StringUtils.isEmpty(metadata) || StringUtils.isEmpty(timeout)) {
updateStateMapWithError(uuid,"signature, metadata and expires are compulsory fields.");
throw new InvalidParameterValueException("signature, metadata and expires are compulsory fields.");
}
+ //check that contentLength exists and is greater than zero
+ if (contentLength <= 0) {
+ throw new InvalidParameterValueException("content length is not set in the request or has invalid value.");
+ }
+
//validate signature
String fullUrl = "https://" + hostname + "/upload/" + uuid;
String computedSignature = EncryptionUtil.generateSignature(metadata + fullUrl + timeout, getPostUploadPSK());
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b16520bc/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
index 46ace7c..15a6ef2 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
@@ -31,6 +31,7 @@ public class UploadEntity {
private long entityId;
private String chksum;
private long physicalSize;
+ private int maxSizeInGB;
public static enum ResourceType {
VOLUME, TEMPLATE
@@ -172,5 +173,11 @@ public class UploadEntity {
return physicalSize;
}
+ public int getMaxSizeInGB() {
+ return maxSizeInGB;
+ }
+ public void setMaxSizeInGB(int maxSizeInGB) {
+ this.maxSizeInGB = maxSizeInGB;
+ }
}