You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by mc...@apache.org on 2013/10/05 02:39:21 UTC
[1/2] git commit: updated refs/heads/4.2 to b63270f
Updated Branches:
refs/heads/4.2 2976d9c79 -> b63270f1d
CLOUDSTACK-4816: Make S3 upload multipart or singlepart configurable.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/975422a9
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/975422a9
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/975422a9
Branch: refs/heads/4.2
Commit: 975422a9500f17c5b595e8c4c6075b3b49463337
Parents: 2976d9c
Author: Min Chen <mi...@citrix.com>
Authored: Fri Oct 4 16:04:36 2013 -0700
Committer: Min Chen <mi...@citrix.com>
Committed: Fri Oct 4 17:38:17 2013 -0700
----------------------------------------------------------------------
api/src/com/cloud/agent/api/to/S3TO.java | 14 ++++-
.../storage/template/S3TemplateDownloader.java | 25 ++++-----
.../driver/S3ImageStoreDriverImpl.java | 5 +-
server/src/com/cloud/configuration/Config.java | 1 +
.../resource/NfsSecondaryStorageResource.java | 7 ++-
utils/src/com/cloud/utils/S3Utils.java | 56 ++++++++++++++++++++
6 files changed, 92 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/api/src/com/cloud/agent/api/to/S3TO.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java
index b1b692a..ab08a69 100644
--- a/api/src/com/cloud/agent/api/to/S3TO.java
+++ b/api/src/com/cloud/agent/api/to/S3TO.java
@@ -39,6 +39,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private Integer socketTimeout;
private Date created;
private boolean enableRRS;
+ private boolean multipartEnabled;
public S3TO() {
@@ -50,7 +51,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
final String secretKey, final String endPoint,
final String bucketName, final Boolean httpsFlag,
final Integer connectionTimeout, final Integer maxErrorRetry,
- final Integer socketTimeout, final Date created, final boolean enableRRS) {
+ final Integer socketTimeout, final Date created, final boolean enableRRS, final boolean multipart) {
super();
@@ -66,6 +67,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.socketTimeout = socketTimeout;
this.created = created;
this.enableRRS = enableRRS;
+ this.multipartEnabled = multipart;
}
@@ -268,7 +270,6 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
}
-
public boolean getEnableRRS() {
return enableRRS;
}
@@ -277,5 +278,14 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.enableRRS = enableRRS;
}
+ public boolean isMultipartEnabled() {
+ return multipartEnabled;
+ }
+
+ public void setMultipartEnabled(boolean multipartEnabled) {
+ this.multipartEnabled = multipartEnabled;
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/core/src/com/cloud/storage/template/S3TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index 340e0db..c394368 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -43,15 +43,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import com.amazonaws.AmazonClientException;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ProgressEvent;
import com.amazonaws.services.s3.model.ProgressListener;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.StorageClass;
-import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.Upload;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.S3TO;
import com.cloud.utils.Pair;
@@ -226,9 +222,6 @@ public class S3TemplateDownloader implements TemplateDownloader {
// compute s3 key
s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR);
- // multi-part upload using S3 api to handle > 5G input stream
- TransferManager tm = new TransferManager(S3Utils.acquireClient(s3));
-
// download using S3 API
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(remoteSize);
@@ -261,11 +254,19 @@ public class S3TemplateDownloader implements TemplateDownloader {
}
});
- // TransferManager processes all transfers asynchronously,
- // so this call will return immediately.
- Upload upload = tm.upload(putObjectRequest);
-
- upload.waitForCompletion();
+
+ if ( s3.isMultipartEnabled()){
+ // use TransferManager to do multipart upload
+ S3Utils.mputObject(s3, putObjectRequest);
+ } else{
+ // single part upload, with 5GB limit in Amazon
+ S3Utils.putObject(s3, putObjectRequest);
+ while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED &&
+ status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
+ status != TemplateDownloader.Status.ABORTED) {
+ // wait for completion
+ }
+ }
// finished or aborted
Date finish = new Date();
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
index ac681cc..117beee 100644
--- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
+++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
@@ -64,7 +64,10 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
details.get(ApiConstants.S3_SOCKET_TIMEOUT) == null ? null : Integer.valueOf(details
.get(ApiConstants.S3_SOCKET_TIMEOUT)), imgStore.getCreated(),
_configDao.getValue(Config.S3EnableRRS.toString()) == null ? false : Boolean.parseBoolean(_configDao
- .getValue(Config.S3EnableRRS.toString())));
+ .getValue(Config.S3EnableRRS.toString())),
+ _configDao.getValue(Config.S3EnableMultiPartUpload.toString()) == null ? true : Boolean.parseBoolean(_configDao
+ .getValue(Config.S3EnableMultiPartUpload.toString()))
+ );
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/server/src/com/cloud/configuration/Config.java
----------------------------------------------------------------------
diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java
index d4f9607..613a882 100755
--- a/server/src/com/cloud/configuration/Config.java
+++ b/server/src/com/cloud/configuration/Config.java
@@ -425,6 +425,7 @@ public enum Config {
// object store
S3EnableRRS("Advanced", ManagementServer.class, Boolean.class, "s3.rrs.enabled", "false", "enable s3 reduced redundancy storage", null),
+ S3EnableMultiPartUpload("Advanced", ManagementServer.class, Boolean.class, "s3.multipart.enabled", "true", "enable s3 multipart upload", null),
// VMSnapshots
VMSnapshotMax("Advanced", VMSnapshotManager.class, Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a vm", null),
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
index 3a563cb..a5a9f2b 100755
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
+++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
@@ -17,6 +17,7 @@
package org.apache.cloudstack.storage.resource;
import static com.cloud.utils.S3Utils.putFile;
+import static com.cloud.utils.S3Utils.mputFile;
import static com.cloud.utils.StringUtils.join;
import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
import static java.lang.String.format;
@@ -821,7 +822,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
ImageFormat format = this.getTemplateFormat(srcFile.getName());
String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
- putFile(s3, srcFile, bucket, key);
+ if (s3.isMultipartEnabled()){
+ mputFile(s3, srcFile, bucket, key);
+ } else{
+ putFile(s3, srcFile, bucket, key);
+ }
DataTO retObj = null;
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/975422a9/utils/src/com/cloud/utils/S3Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java
index 5ee5783..ce4d4b7 100644
--- a/utils/src/com/cloud/utils/S3Utils.java
+++ b/utils/src/com/cloud/utils/S3Utils.java
@@ -48,6 +48,7 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.HttpMethod;
import com.amazonaws.auth.AWSCredentials;
@@ -61,6 +62,9 @@ import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.Upload;
+
import com.cloud.utils.exception.CloudRuntimeException;
public final class S3Utils {
@@ -171,6 +175,58 @@ public final class S3Utils {
}
+ // multi-part upload file
+ public static void mputFile(final ClientOptions clientOptions,
+ final File sourceFile, final String bucketName, final String key) throws InterruptedException {
+
+ assert clientOptions != null;
+ assert sourceFile != null;
+ assert !isBlank(bucketName);
+ assert !isBlank(key);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(format("Multipart sending file %1$s as S3 object %2$s in "
+ + "bucket %3$s", sourceFile.getName(), key, bucketName));
+ }
+ TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
+ Upload upload = tm.upload(bucketName, key, sourceFile);
+ upload.waitForCompletion();
+ }
+
+ // multi-part upload object
+ public static void mputObject(final ClientOptions clientOptions,
+ final InputStream sourceStream, final String bucketName, final String key) throws InterruptedException {
+
+ assert clientOptions != null;
+ assert sourceStream != null;
+ assert !isBlank(bucketName);
+ assert !isBlank(key);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(format("Multipart sending stream as S3 object %1$s in "
+ + "bucket %2$s", key, bucketName));
+ }
+ TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
+ Upload upload = tm.upload(bucketName, key, sourceStream, null);
+ upload.waitForCompletion();
+ }
+
+ // multi-part upload object
+ public static void mputObject(final ClientOptions clientOptions,
+ final PutObjectRequest req) throws InterruptedException {
+
+ assert clientOptions != null;
+ assert req != null;
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Multipart sending object to S3 using PutObjectRequest");
+ }
+ TransferManager tm = new TransferManager(S3Utils.acquireClient(clientOptions));
+ Upload upload = tm.upload(req);
+ upload.waitForCompletion();
+
+ }
+
public static void setObjectAcl(final ClientOptions clientOptions, final String bucketName, final String key,
final CannedAccessControlList acl) {
[2/2] git commit: updated refs/heads/4.2 to b63270f
Posted by mc...@apache.org.
CLOUDSTACK-4816: Add global configure s3.multipart.enabled
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/b63270f1
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/b63270f1
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/b63270f1
Branch: refs/heads/4.2
Commit: b63270f1d355458e9f5f2543d4320c5c832c41fe
Parents: 975422a
Author: Min Chen <mi...@citrix.com>
Authored: Fri Oct 4 17:27:04 2013 -0700
Committer: Min Chen <mi...@citrix.com>
Committed: Fri Oct 4 17:38:40 2013 -0700
----------------------------------------------------------------------
.../com/cloud/upgrade/dao/Upgrade420to421.java | 10 ++++++--
setup/db/db/schema-420to421.sql | 24 ++++++++++++++++++++
2 files changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b63270f1/engine/schema/src/com/cloud/upgrade/dao/Upgrade420to421.java
----------------------------------------------------------------------
diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade420to421.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade420to421.java
index 6707eb9..27704e8 100755
--- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade420to421.java
+++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade420to421.java
@@ -26,6 +26,7 @@ import java.sql.SQLException;
import org.apache.log4j.Logger;
import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
public class Upgrade420to421 implements DbUpgrade {
final static Logger s_logger = Logger.getLogger(Upgrade420to421.class);
@@ -33,7 +34,7 @@ public class Upgrade420to421 implements DbUpgrade {
@Override
public String[] getUpgradableVersionRange() {
- return new String[] { "4.2.0" };
+ return new String[] { "4.2.0", "4.2.1" };
}
@Override
@@ -48,7 +49,12 @@ public class Upgrade420to421 implements DbUpgrade {
@Override
public File[] getPrepareScripts() {
- return null;
+ String script = Script.findScript("", "db/schema-420to421.sql");
+ if (script == null) {
+ throw new CloudRuntimeException("Unable to find db/schema-420to421.sql");
+ }
+
+ return new File[] { new File(script) };
}
@Override
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b63270f1/setup/db/db/schema-420to421.sql
----------------------------------------------------------------------
diff --git a/setup/db/db/schema-420to421.sql b/setup/db/db/schema-420to421.sql
new file mode 100644
index 0000000..1e6341e
--- /dev/null
+++ b/setup/db/db/schema-420to421.sql
@@ -0,0 +1,24 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.2.0 to 4.2.1;
+--;
+
+
+INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 's3.multipart.enabled', 'true', 'enable s3 multipart upload');
+