You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by mc...@apache.org on 2013/10/17 02:57:14 UTC
git commit: updated refs/heads/4.2 to d17024c
Updated Branches:
refs/heads/4.2 1a907eb4e -> d17024c09
CLOUDSTACK-4816:provide configurable option to choose single vs
multipart upload to S3 object storage based on object size.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/d17024c0
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/d17024c0
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/d17024c0
Branch: refs/heads/4.2
Commit: d17024c09baf4ecb564674dc272268cdc26bcca2
Parents: 1a907eb
Author: Min Chen <mi...@citrix.com>
Authored: Wed Oct 16 17:56:19 2013 -0700
Committer: Min Chen <mi...@citrix.com>
Committed: Wed Oct 16 17:56:19 2013 -0700
----------------------------------------------------------------------
api/src/com/cloud/agent/api/to/S3TO.java | 31 +++++++++++++++-----
.../storage/template/S3TemplateDownloader.java | 3 +-
.../driver/S3ImageStoreDriverImpl.java | 12 ++++++--
server/src/com/cloud/configuration/Config.java | 3 +-
.../resource/NfsSecondaryStorageResource.java | 4 ++-
setup/db/db/schema-420to421.sql | 3 +-
utils/src/com/cloud/utils/S3Utils.java | 4 +++
7 files changed, 46 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/api/src/com/cloud/agent/api/to/S3TO.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java
index ab08a69..ea7564d 100644
--- a/api/src/com/cloud/agent/api/to/S3TO.java
+++ b/api/src/com/cloud/agent/api/to/S3TO.java
@@ -40,6 +40,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private Date created;
private boolean enableRRS;
private boolean multipartEnabled;
+ private long maxSingleUploadSizeInBytes;
public S3TO() {
@@ -51,7 +52,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
final String secretKey, final String endPoint,
final String bucketName, final Boolean httpsFlag,
final Integer connectionTimeout, final Integer maxErrorRetry,
- final Integer socketTimeout, final Date created, final boolean enableRRS, final boolean multipart) {
+ final Integer socketTimeout, final Date created, final boolean enableRRS, final long maxUploadSize) {
super();
@@ -67,7 +68,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.socketTimeout = socketTimeout;
this.created = created;
this.enableRRS = enableRRS;
- this.multipartEnabled = multipart;
+ this.maxSingleUploadSizeInBytes = maxUploadSize;
}
@@ -278,14 +279,28 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.enableRRS = enableRRS;
}
- public boolean isMultipartEnabled() {
- return multipartEnabled;
+ public long getMaxSingleUploadSizeInBytes() {
+ return maxSingleUploadSizeInBytes;
}
- public void setMultipartEnabled(boolean multipartEnabled) {
- this.multipartEnabled = multipartEnabled;
+ public void setMaxSingleUploadSizeInBytes(long maxSingleUploadSizeInBytes) {
+ this.maxSingleUploadSizeInBytes = maxSingleUploadSizeInBytes;
}
-
-
+ public boolean getSingleUpload(long objSize){
+ if ( maxSingleUploadSizeInBytes < 0 ){
+ // always use single part upload
+ return true;
+ } else if ( maxSingleUploadSizeInBytes == 0 ){
+ // always use multi part upload
+ return false;
+ } else {
+ // check object size to set flag
+ if (objSize < maxSingleUploadSizeInBytes){
+ return true;
+ } else{
+ return false;
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/core/src/com/cloud/storage/template/S3TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index c394368..9c2b628 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -255,7 +255,8 @@ public class S3TemplateDownloader implements TemplateDownloader {
});
- if ( s3.isMultipartEnabled()){
+
+ if ( !s3.getSingleUpload(remoteSize) ){
// use TransferManager to do multipart upload
S3Utils.mputObject(s3, putObjectRequest);
} else{
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
index 117beee..e0afe51 100644
--- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
+++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
@@ -65,13 +65,21 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
.get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(),
_configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao
.getValue(Config.S3EnableRRS.toString())),
- _configDao.getValue(Config.S3EnableMultiPartUpload.toString()) == null ? true : Boolean.parseBoolean(_configDao
- .getValue(Config.S3EnableMultiPartUpload.toString()))
+ getMaxSingleUploadSizeInBytes()
);
}
+ private long getMaxSingleUploadSizeInBytes() {
+ try {
+ return Long.parseLong(_configDao.getValue(Config.S3MaxSingleUploadSize.toString())) * 1024L * 1024L * 1024L;
+ } catch (NumberFormatException e) {
+ // use default 5GB
+ return 5L * 1024L * 1024L * 1024L;
+ }
+ }
+
@Override
public String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject) {
// for S3, no need to do anything, just return template url for
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/server/src/com/cloud/configuration/Config.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java
index eac3b75..72fe5dd 100755
--- a/server/src/com/cloud/configuration/Config.java
+++ b/server/src/com/cloud/configuration/Config.java
@@ -426,7 +426,8 @@ public enum Config {
// object store
S3EnableRRS("Advanced", ManagementServer.class, Boolean.class, "s3.rrs.enabled", "false", "enable s3 reduced redundancy storage", null),
- S3EnableMultiPartUpload("Advanced", ManagementServer.class, Boolean.class, "s3.multipart.enabled", "true", "enable s3 multipart upload", null),
+ S3MaxSingleUploadSize("Advanced", ManagementServer.class, Integer.class, "s3.singleupload.max.size", "5", "The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then it means always use multi-part upload to upload object to S3. " +
+ "If it is set to -1, then it means always use single-part upload to upload object to S3. ", null),
// VMSnapshots
VMSnapshotMax("Advanced", VMSnapshotManager.class, Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a vm", null),
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index a5a9f2b..bd2378c 100755
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -820,9 +820,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
}
+
+ long srcSize = srcFile.length();
ImageFormat format = this.getTemplateFormat(srcFile.getName());
String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
- if (s3.isMultipartEnabled()){
+ if (!s3.getSingleUpload(srcSize)){
mputFile(s3, srcFile, bucket, key);
} else{
putFile(s3, srcFile, bucket, key);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/setup/db/db/schema-420to421.sql
----------------------------------------------------------------------
diff --git a/setup/db/db/schema-420to421.sql b/setup/db/db/schema-420to421.sql
index 4dfe4e6..abe97a4 100644
--- a/setup/db/db/schema-420to421.sql
+++ b/setup/db/db/schema-420to421.sql
@@ -20,7 +20,8 @@
--;
-INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.multipart.enabled', 'true', 'enable s3 multipart upload');
+INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.singleupload.max.size', '5',
+ 'The maximum size limit for S3 single part upload API(in GB). If it is set to 0, then it means always use multi-part upload to upload object to S3. If it is set to -1, then it means always use single-part upload to upload object to S3.');
INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Storage", 'DEFAULT', 'management-server', "enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage during HA");
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/d17024c0/utils/src/com/cloud/utils/S3Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java
index ce4d4b7..7918a20 100644
--- a/utils/src/com/cloud/utils/S3Utils.java
+++ b/utils/src/com/cloud/utils/S3Utils.java
@@ -171,6 +171,10 @@ public final class S3Utils {
assert clientOptions != null;
assert req != null;
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(format("Sending stream as S3 object using PutObjectRequest"));
+ }
+
acquireClient(clientOptions).putObject(req);
}