You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by re...@apache.org on 2015/11/20 22:28:07 UTC
[2/3] git commit: updated refs/heads/master to 7665bdc
CLOUDSTACK-9062: Improve S3 implementation.
The S3 implementation is far from finished, this commit focusses on the bases.
- Upgrade AWS SDK to latest version.
- Rewrite S3 Template downloader.
- Rewrite S3Utils utility class.
- Improve addImageStoreS3 API command.
- Split various classes for convenience.
- Various minor improvements and code optimalisations.
A side effect of the new AWS SDK is that it, by default, uses the V4 signature. Therefore I added an option to specify the Signer, so it stays compatible with previous versions.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/5c0366c9
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/5c0366c9
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/5c0366c9
Branch: refs/heads/master
Commit: 5c0366c99e60a1c2e3d4d03045026fce55165be1
Parents: fe2917e
Author: Boris Schrijver <bo...@pcextreme.nl>
Authored: Fri Nov 13 02:19:24 2015 +0100
Committer: Boris Schrijver <bo...@pcextreme.nl>
Committed: Thu Nov 19 15:29:10 2015 +0100
----------------------------------------------------------------------
agent/conf/log4j-cloud.xml.in | 12 +-
api/src/com/cloud/agent/api/storage/Proxy.java | 72 ---
api/src/com/cloud/agent/api/to/S3TO.java | 215 +++----
api/src/com/cloud/storage/StorageService.java | 24 +-
.../org/apache/cloudstack/api/ApiConstants.java | 3 +
.../admin/storage/AddImageStoreS3CMD.java | 276 +++++++++
.../api/command/admin/storage/AddS3Cmd.java | 264 --------
.../api/command/admin/storage/ListS3sCmd.java | 55 --
.../template/RegisterTemplateCmdByAdmin.java | 2 +-
client/tomcatconf/commands.properties.in | 5 +-
client/tomcatconf/log4j-cloud.xml.in | 10 +
core/src/com/cloud/resource/ServerResource.java | 2 +-
.../template/HttpTemplateDownloader.java | 2 +-
.../storage/template/S3TemplateDownloader.java | 483 ++++++---------
.../storage/template/TemplateDownloader.java | 33 +-
.../storage/command/DownloadCommand.java | 2 +-
.../api/storage/DataStoreProvider.java | 16 +-
.../storage/image/TemplateServiceImpl.java | 2 +-
.../datastore/ObjectInDataStoreManager.java | 10 +-
.../storage/image/BaseImageStoreDriverImpl.java | 2 +-
framework/rest/pom.xml | 10 +-
packaging/centos7/tomcat7/log4j-cloud.xml | 10 +
.../kvm/storage/KVMStorageProcessor.java | 18 +-
.../resource/XenServerStorageProcessor.java | 4 +-
plugins/storage/image/s3/pom.xml | 2 +-
.../driver/S3ImageStoreDriverImpl.java | 31 +-
.../lifecycle/S3ImageStoreLifeCycleImpl.java | 6 +-
.../provider/S3ImageStoreProviderImpl.java | 7 +-
pom.xml | 3 +-
scripts/vm/hypervisor/xenserver/s3xenserver | 2 +-
server/conf/log4j-cloud.xml.in | 10 +
.../com/cloud/server/ManagementServerImpl.java | 6 +-
.../com/cloud/storage/StorageManagerImpl.java | 25 +-
.../storage/download/DownloadMonitorImpl.java | 2 +-
.../resource/NfsSecondaryStorageResource.java | 30 +-
.../resource/SecondaryStorageResource.java | 3 +-
.../SecondaryStorageResourceHandler.java | 2 +
.../storage/template/DownloadManager.java | 2 +-
.../storage/template/DownloadManagerImpl.java | 2 +-
systemvm/conf.dom0/log4j-cloud.xml.in | 10 +
systemvm/conf/log4j-cloud.xml | 10 +
utils/pom.xml | 5 +
.../src/main/java/com/cloud/utils/S3Utils.java | 619 -------------------
.../java/com/cloud/utils/net/HTTPUtils.java | 143 +++++
.../main/java/com/cloud/utils/net/Proxy.java | 69 +++
.../cloud/utils/storage/S3/ClientOptions.java | 42 ++
.../utils/storage/S3/FileNamingStrategy.java | 25 +
.../utils/storage/S3/ObjectNamingStrategy.java | 27 +
.../com/cloud/utils/storage/S3/S3Utils.java | 216 +++++++
.../utils/rest/RESTServiceConnectorTest.java | 4 +-
50 files changed, 1300 insertions(+), 1535 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/agent/conf/log4j-cloud.xml.in
----------------------------------------------------------------------
diff --git a/agent/conf/log4j-cloud.xml.in b/agent/conf/log4j-cloud.xml.in
index 9e6c646..1222801 100644
--- a/agent/conf/log4j-cloud.xml.in
+++ b/agent/conf/log4j-cloud.xml.in
@@ -77,7 +77,17 @@ under the License.
</category>
<category name="net">
- <priority value="INFO"/>
+ <priority value="INFO"/>
+ </category>
+
+ <!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
+ <category name="com.amazonaws">
+ <priority value="INFO"/>
+ </category>
+
+ <!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
+ <category name="httpclient.wire">
+ <priority value="INFO"/>
</category>
<!-- ======================= -->
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/com/cloud/agent/api/storage/Proxy.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/storage/Proxy.java b/api/src/com/cloud/agent/api/storage/Proxy.java
deleted file mode 100644
index 5adc114..0000000
--- a/api/src/com/cloud/agent/api/storage/Proxy.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package com.cloud.agent.api.storage;
-
-import java.net.URI;
-
-/**
- * Download Proxy
- */
-public class Proxy {
- private String _host;
- private int _port;
- private String _userName;
- private String _password;
-
- public Proxy() {
-
- }
-
- public Proxy(String host, int port, String userName, String password) {
- this._host = host;
- this._port = port;
- this._userName = userName;
- this._password = password;
- }
-
- public Proxy(URI uri) {
- this._host = uri.getHost();
- this._port = uri.getPort() == -1 ? 3128 : uri.getPort();
- String userInfo = uri.getUserInfo();
- if (userInfo != null) {
- String[] tokens = userInfo.split(":");
- if (tokens.length == 1) {
- this._userName = userInfo;
- this._password = "";
- } else if (tokens.length == 2) {
- this._userName = tokens[0];
- this._password = tokens[1];
- }
- }
- }
-
- public String getHost() {
- return _host;
- }
-
- public int getPort() {
- return _port;
- }
-
- public String getUserName() {
- return _userName;
- }
-
- public String getPassword() {
- return _password;
- }
-}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/com/cloud/agent/api/to/S3TO.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/agent/api/to/S3TO.java b/api/src/com/cloud/agent/api/to/S3TO.java
index a46d609..ec6bc02 100644
--- a/api/src/com/cloud/agent/api/to/S3TO.java
+++ b/api/src/com/cloud/agent/api/to/S3TO.java
@@ -21,9 +21,9 @@ import java.util.Date;
import com.cloud.agent.api.LogLevel;
import com.cloud.agent.api.LogLevel.Log4jLevel;
import com.cloud.storage.DataStoreRole;
-import com.cloud.utils.S3Utils;
+import com.cloud.utils.storage.S3.ClientOptions;
-public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
+public final class S3TO implements ClientOptions, DataStoreTO {
private Long id;
private String uuid;
@@ -33,6 +33,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private String secretKey;
private String endPoint;
private String bucketName;
+ private String signer;
private Boolean httpsFlag;
private Boolean useTCPKeepAlive;
private Integer connectionTimeout;
@@ -44,17 +45,9 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
private long maxSingleUploadSizeInBytes;
private static final String pathSeparator = "/";
- public S3TO() {
-
- super();
-
- }
-
public S3TO(final Long id, final String uuid, final String accessKey, final String secretKey, final String endPoint, final String bucketName,
- final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout, final Date created,
- final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
-
- super();
+ final String signer, final Boolean httpsFlag, final Integer connectionTimeout, final Integer maxErrorRetry, final Integer socketTimeout,
+ final Date created, final boolean enableRRS, final long maxUploadSize, final Integer connectionTtl, final Boolean useTCPKeepAlive) {
this.id = id;
this.uuid = uuid;
@@ -62,6 +55,7 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
this.secretKey = secretKey;
this.endPoint = endPoint;
this.bucketName = bucketName;
+ this.signer = signer;
this.httpsFlag = httpsFlag;
this.connectionTimeout = connectionTimeout;
this.maxErrorRetry = maxErrorRetry;
@@ -74,98 +68,6 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
}
- @Override
- public boolean equals(final Object thatObject) {
-
- if (this == thatObject) {
- return true;
- }
- if (thatObject == null || getClass() != thatObject.getClass()) {
- return false;
- }
-
- final S3TO thatS3TO = (S3TO)thatObject;
-
- if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
- return false;
- }
-
- if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
- return false;
- }
-
- if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
- return false;
- }
-
- if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
- return false;
- }
-
- if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
- return false;
- }
-
- if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
- return false;
- }
-
- if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
- return false;
- }
-
- if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
- return false;
- }
-
- if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
- return false;
- }
-
- if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
- return false;
- }
-
- if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
- return false;
- }
-
- if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
- return false;
- }
-
- if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
- return false;
- }
-
- if (enableRRS != thatS3TO.enableRRS) {
- return false;
- }
-
- return true;
-
- }
-
- @Override
- public int hashCode() {
-
- int result = id != null ? id.hashCode() : 0;
-
- result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
- result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
- result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
- result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
- result = 31 * result + (httpsFlag ? 1 : 0);
- result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
- result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
- result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
- result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
- result = 31 * result + (useTCPKeepAlive ? 1 : 0);
-
- return result;
-
- }
-
public Long getId() {
return this.id;
}
@@ -224,6 +126,15 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
}
@Override
+ public String getSigner() {
+ return this.signer;
+ }
+
+ public void setSigner(final String signer) {
+ this.signer = signer;
+ }
+
+ @Override
public Boolean isHttps() {
return this.httpsFlag;
}
@@ -327,4 +238,100 @@ public final class S3TO implements S3Utils.ClientOptions, DataStoreTO {
public String getPathSeparator() {
return pathSeparator;
}
+
+ @Override
+ public boolean equals(final Object thatObject) {
+
+ if (this == thatObject) {
+ return true;
+ }
+ if (thatObject == null || getClass() != thatObject.getClass()) {
+ return false;
+ }
+
+ final S3TO thatS3TO = (S3TO)thatObject;
+
+ if (httpsFlag != null ? !httpsFlag.equals(thatS3TO.httpsFlag) : thatS3TO.httpsFlag != null) {
+ return false;
+ }
+
+ if (accessKey != null ? !accessKey.equals(thatS3TO.accessKey) : thatS3TO.accessKey != null) {
+ return false;
+ }
+
+ if (connectionTimeout != null ? !connectionTimeout.equals(thatS3TO.connectionTimeout) : thatS3TO.connectionTimeout != null) {
+ return false;
+ }
+
+ if (endPoint != null ? !endPoint.equals(thatS3TO.endPoint) : thatS3TO.endPoint != null) {
+ return false;
+ }
+
+ if (id != null ? !id.equals(thatS3TO.id) : thatS3TO.id != null) {
+ return false;
+ }
+
+ if (uuid != null ? !uuid.equals(thatS3TO.uuid) : thatS3TO.uuid != null) {
+ return false;
+ }
+
+ if (maxErrorRetry != null ? !maxErrorRetry.equals(thatS3TO.maxErrorRetry) : thatS3TO.maxErrorRetry != null) {
+ return false;
+ }
+
+ if (secretKey != null ? !secretKey.equals(thatS3TO.secretKey) : thatS3TO.secretKey != null) {
+ return false;
+ }
+
+ if (socketTimeout != null ? !socketTimeout.equals(thatS3TO.socketTimeout) : thatS3TO.socketTimeout != null) {
+ return false;
+ }
+
+ if (connectionTtl != null ? !connectionTtl.equals(thatS3TO.connectionTtl) : thatS3TO.connectionTtl != null) {
+ return false;
+ }
+
+ if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatS3TO.useTCPKeepAlive) : thatS3TO.useTCPKeepAlive != null) {
+ return false;
+ }
+
+ if (bucketName != null ? !bucketName.equals(thatS3TO.bucketName) : thatS3TO.bucketName != null) {
+ return false;
+ }
+
+ if (signer != null ? !signer.equals(thatS3TO.signer) : thatS3TO.signer != null) {
+ return false;
+ }
+
+ if (created != null ? !created.equals(thatS3TO.created) : thatS3TO.created != null) {
+ return false;
+ }
+
+ if (enableRRS != thatS3TO.enableRRS) {
+ return false;
+ }
+
+ return true;
+
+ }
+
+ @Override
+ public int hashCode() {
+
+ int result = id != null ? id.hashCode() : 0;
+
+ result = 31 * result + (accessKey != null ? accessKey.hashCode() : 0);
+ result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
+ result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
+ result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
+ result = 31 * result + (signer != null ? signer.hashCode() : 0);
+ result = 31 * result + (httpsFlag ? 1 : 0);
+ result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
+ result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
+ result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
+ result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
+ result = 31 * result + (useTCPKeepAlive ? 1 : 0);
+
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/com/cloud/storage/StorageService.java
----------------------------------------------------------------------
diff --git a/api/src/com/cloud/storage/StorageService.java b/api/src/com/cloud/storage/StorageService.java
index 7fc1e16..e40b1e6 100644
--- a/api/src/com/cloud/storage/StorageService.java
+++ b/api/src/com/cloud/storage/StorageService.java
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
package com.cloud.storage;
import java.net.UnknownHostException;
@@ -38,14 +39,14 @@ public interface StorageService {
* Create StoragePool based on uri
*
* @param cmd
- * the command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
+ * The command object that specifies the zone, cluster/pod, URI, details, etc. to use to create the
* storage pool.
* @return
+ * The StoragePool created.
* @throws ResourceInUseException
* @throws IllegalArgumentException
* @throws UnknownHostException
* @throws ResourceUnavailableException
- * TODO
*/
StoragePool createPool(CreateStoragePoolCmd cmd) throws ResourceInUseException, IllegalArgumentException, UnknownHostException, ResourceUnavailableException;
@@ -63,15 +64,13 @@ public interface StorageService {
/**
* Enable maintenance for primary storage
*
- * @param cmd
- * - the command specifying primaryStorageId
+ * @param primaryStorageId
+ * - the primaryStorageId
* @return the primary storage pool
* @throws ResourceUnavailableException
- * TODO
* @throws InsufficientCapacityException
- * TODO
*/
- public StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
+ StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
/**
* Complete maintenance for primary storage
@@ -80,19 +79,18 @@ public interface StorageService {
* - the command specifying primaryStorageId
* @return the primary storage pool
* @throws ResourceUnavailableException
- * TODO
*/
- public StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
+ StoragePool cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException;
- public StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
+ StoragePool updateStoragePool(UpdateStoragePoolCmd cmd) throws IllegalArgumentException;
- public StoragePool getStoragePool(long id);
+ StoragePool getStoragePool(long id);
boolean deleteImageStore(DeleteImageStoreCmd cmd);
boolean deleteSecondaryStagingStore(DeleteSecondaryStagingStoreCmd cmd);
- public ImageStore discoverImageStore(String name, String url, String providerName, Long dcId, Map details) throws IllegalArgumentException, DiscoveryException,
+ ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException;
@@ -107,7 +105,7 @@ public interface StorageService {
* @throws DiscoveryException
* @throws InvalidParameterValueException
*/
- public ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
+ ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException,
InvalidParameterValueException;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/ApiConstants.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java
index 2728bcc..5365e14 100644
--- a/api/src/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/org/apache/cloudstack/api/ApiConstants.java
@@ -501,6 +501,9 @@ public class ApiConstants {
public static final String S3_SECRET_KEY = "secretkey";
public static final String S3_END_POINT = "endpoint";
public static final String S3_BUCKET_NAME = "bucket";
+ public static final String S3_SIGNER = "s3signer";
+ public static final String S3_V3_SIGNER = "S3SignerType";
+ public static final String S3_V4_SIGNER = "AWSS3V4SignerType";
public static final String S3_HTTPS_FLAG = "usehttps";
public static final String S3_CONNECTION_TIMEOUT = "connectiontimeout";
public static final String S3_CONNECTION_TTL = "connectionttl";
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
new file mode 100644
index 0000000..34ff171
--- /dev/null
+++ b/api/src/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.api.command.admin.storage;
+
+import static com.cloud.user.Account.ACCOUNT_ID_SYSTEM;
+import static org.apache.cloudstack.api.ApiConstants.S3_ACCESS_KEY;
+import static org.apache.cloudstack.api.ApiConstants.S3_BUCKET_NAME;
+import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TIMEOUT;
+import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TTL;
+import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT;
+import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG;
+import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY;
+import static org.apache.cloudstack.api.ApiConstants.S3_SIGNER;
+import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY;
+import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT;
+import static org.apache.cloudstack.api.ApiConstants.S3_USE_TCP_KEEPALIVE;
+import static org.apache.cloudstack.api.BaseCmd.CommandType.BOOLEAN;
+import static org.apache.cloudstack.api.BaseCmd.CommandType.INTEGER;
+import static org.apache.cloudstack.api.BaseCmd.CommandType.STRING;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.cloud.utils.storage.S3.ClientOptions;
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.ImageStoreResponse;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.DiscoveryException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.storage.ImageStore;
+
+@APICommand(name = "addImageStoreS3", description = "Adds S3 Image Store", responseObject = ImageStoreResponse.class, since = "4.7.0",
+ requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
+public final class AddImageStoreS3CMD extends BaseCmd implements ClientOptions {
+ public static final Logger s_logger = Logger.getLogger(AddImageStoreS3CMD.class.getName());
+
+ private static final String s_name = "addImageStoreS3Response";
+
+ @Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, description = "S3 access key")
+ private String accessKey;
+
+ @Parameter(name = S3_SECRET_KEY, type = STRING, required = true, description = "S3 secret key")
+ private String secretKey;
+
+ @Parameter(name = S3_END_POINT, type = STRING, required = true, description = "S3 endpoint")
+ private String endPoint;
+
+ @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "Name of the storage bucket")
+ private String bucketName;
+
+ @Parameter(name = S3_SIGNER, type = STRING, required = false, description = "Signer Algorithm to use, either S3SignerType or AWSS3V4SignerType")
+ private String signer;
+
+ @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "Use HTTPS instead of HTTP")
+ private Boolean httpsFlag;
+
+ @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "Connection timeout (milliseconds)")
+ private Integer connectionTimeout;
+
+ @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "Maximum number of times to retry on error")
+ private Integer maxErrorRetry;
+
+ @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "Socket timeout (milliseconds)")
+ private Integer socketTimeout;
+
+ @Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "Connection TTL (milliseconds)")
+ private Integer connectionTtl;
+
+ @Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "Whether TCP keep-alive is used")
+ private Boolean useTCPKeepAlive;
+
+ @Override
+ public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
+ ResourceAllocationException, NetworkRuleConflictException {
+
+ Map<String, String> dm = new HashMap();
+
+ dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey());
+ dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey());
+ dm.put(ApiConstants.S3_END_POINT, getEndPoint());
+ dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName());
+
+ if (getSigner() != null && (getSigner().equals(ApiConstants.S3_V3_SIGNER) || getSigner().equals(ApiConstants.S3_V4_SIGNER))) {
+ dm.put(ApiConstants.S3_SIGNER, getSigner());
+ }
+ if (isHttps() != null) {
+ dm.put(ApiConstants.S3_HTTPS_FLAG, isHttps().toString());
+ }
+ if (getConnectionTimeout() != null) {
+ dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString());
+ }
+ if (getMaxErrorRetry() != null) {
+ dm.put(ApiConstants.S3_MAX_ERROR_RETRY, getMaxErrorRetry().toString());
+ }
+ if (getSocketTimeout() != null) {
+ dm.put(ApiConstants.S3_SOCKET_TIMEOUT, getSocketTimeout().toString());
+ }
+ if (getConnectionTtl() != null) {
+ dm.put(ApiConstants.S3_CONNECTION_TTL, getConnectionTtl().toString());
+ }
+ if (getUseTCPKeepAlive() != null) {
+ dm.put(ApiConstants.S3_USE_TCP_KEEPALIVE, getUseTCPKeepAlive().toString());
+ }
+
+ try{
+ ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
+ ImageStoreResponse storeResponse;
+ if (result != null) {
+ storeResponse = _responseGenerator.createImageStoreResponse(result);
+ storeResponse.setResponseName(getCommandName());
+ storeResponse.setObjectName("imagestore");
+ setResponseObject(storeResponse);
+ } else {
+ throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 Image Store.");
+ }
+ } catch (DiscoveryException ex) {
+ s_logger.warn("Exception: ", ex);
+ throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
+ }
+ }
+
+ @Override
+ public String getCommandName() {
+ return s_name;
+ }
+
+ @Override
+ public long getEntityOwnerId() {
+ return ACCOUNT_ID_SYSTEM;
+ }
+
+ public String getAccessKey() {
+ return accessKey;
+ }
+
+ public String getSecretKey() {
+ return secretKey;
+ }
+
+ public String getEndPoint() {
+ return endPoint;
+ }
+
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ public String getSigner() {
+ return signer;
+ }
+
+ public Boolean isHttps() {
+ return httpsFlag;
+ }
+
+ public Integer getConnectionTimeout() {
+ return connectionTimeout;
+ }
+
+ public Integer getMaxErrorRetry() {
+ return maxErrorRetry;
+ }
+
+ public Integer getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ public Integer getConnectionTtl() {
+ return connectionTtl;
+ }
+
+ public Boolean getUseTCPKeepAlive() {
+ return useTCPKeepAlive;
+ }
+
+ @Override
+ public boolean equals(final Object thatObject) {
+
+ if (this == thatObject) {
+ return true;
+ }
+
+ if (thatObject == null || this.getClass() != thatObject.getClass()) {
+ return false;
+ }
+
+ final AddImageStoreS3CMD thatAddS3Cmd = (AddImageStoreS3CMD)thatObject;
+
+ if (httpsFlag != null ? !httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) {
+ return false;
+ }
+
+ if (accessKey != null ? !accessKey.equals(thatAddS3Cmd.accessKey) : thatAddS3Cmd.accessKey != null) {
+ return false;
+ }
+
+ if (connectionTimeout != null ? !connectionTimeout.equals(thatAddS3Cmd.connectionTimeout) : thatAddS3Cmd.connectionTimeout != null) {
+ return false;
+ }
+
+ if (endPoint != null ? !endPoint.equals(thatAddS3Cmd.endPoint) : thatAddS3Cmd.endPoint != null) {
+ return false;
+ }
+
+ if (maxErrorRetry != null ? !maxErrorRetry.equals(thatAddS3Cmd.maxErrorRetry) : thatAddS3Cmd.maxErrorRetry != null) {
+ return false;
+ }
+
+ if (secretKey != null ? !secretKey.equals(thatAddS3Cmd.secretKey) : thatAddS3Cmd.secretKey != null) {
+ return false;
+ }
+
+ if (socketTimeout != null ? !socketTimeout.equals(thatAddS3Cmd.socketTimeout) : thatAddS3Cmd.socketTimeout != null) {
+ return false;
+ }
+
+ if (bucketName != null ? !bucketName.equals(thatAddS3Cmd.bucketName) : thatAddS3Cmd.bucketName != null) {
+ return false;
+ }
+
+ if (connectionTtl != null ? !connectionTtl.equals(thatAddS3Cmd.connectionTtl) : thatAddS3Cmd.connectionTtl != null) {
+ return false;
+ }
+
+ if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatAddS3Cmd.useTCPKeepAlive) : thatAddS3Cmd.useTCPKeepAlive != null) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+
+ int result = accessKey != null ? accessKey.hashCode() : 0;
+ result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
+ result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
+ result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
+ result = 31 * result + (signer != null ? signer.hashCode() : 0);
+ result = 31 * result + (httpsFlag != null && httpsFlag ? 1 : 0);
+ result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
+ result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
+ result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
+ result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
+ result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive ? 1 : 0);
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
deleted file mode 100644
index d54cb75..0000000
--- a/api/src/org/apache/cloudstack/api/command/admin/storage/AddS3Cmd.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cloudstack.api.command.admin.storage;
-
-import static com.cloud.user.Account.ACCOUNT_ID_SYSTEM;
-import static org.apache.cloudstack.api.ApiConstants.S3_ACCESS_KEY;
-import static org.apache.cloudstack.api.ApiConstants.S3_BUCKET_NAME;
-import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TIMEOUT;
-import static org.apache.cloudstack.api.ApiConstants.S3_CONNECTION_TTL;
-import static org.apache.cloudstack.api.ApiConstants.S3_END_POINT;
-import static org.apache.cloudstack.api.ApiConstants.S3_HTTPS_FLAG;
-import static org.apache.cloudstack.api.ApiConstants.S3_MAX_ERROR_RETRY;
-import static org.apache.cloudstack.api.ApiConstants.S3_SECRET_KEY;
-import static org.apache.cloudstack.api.ApiConstants.S3_SOCKET_TIMEOUT;
-import static org.apache.cloudstack.api.ApiConstants.S3_USE_TCP_KEEPALIVE;
-import static org.apache.cloudstack.api.BaseCmd.CommandType.BOOLEAN;
-import static org.apache.cloudstack.api.BaseCmd.CommandType.INTEGER;
-import static org.apache.cloudstack.api.BaseCmd.CommandType.STRING;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.ApiErrorCode;
-import org.apache.cloudstack.api.BaseCmd;
-import org.apache.cloudstack.api.Parameter;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.ImageStoreResponse;
-
-import com.cloud.exception.ConcurrentOperationException;
-import com.cloud.exception.DiscoveryException;
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.NetworkRuleConflictException;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.storage.ImageStore;
-
-@APICommand(name = "addS3", description = "Adds S3", responseObject = ImageStoreResponse.class, since = "4.0.0",
- requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
-public final class AddS3Cmd extends BaseCmd {
- public static final Logger s_logger = Logger.getLogger(AddS3Cmd.class.getName());
-
- private static final String s_name = "adds3response";
-
- @Parameter(name = S3_ACCESS_KEY, type = STRING, required = true, description = "S3 access key")
- private String accessKey;
-
- @Parameter(name = S3_SECRET_KEY, type = STRING, required = true, description = "S3 secret key")
- private String secretKey;
-
- @Parameter(name = S3_END_POINT, type = STRING, required = false, description = "S3 host name")
- private String endPoint;
-
- @Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "name of the template storage bucket")
- private String bucketName;
-
- @Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "connect to the S3 endpoint via HTTPS?")
- private Boolean httpsFlag;
-
- @Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "connection timeout (milliseconds)")
- private Integer connectionTimeout;
-
- @Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "maximum number of times to retry on error")
- private Integer maxErrorRetry;
-
- @Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "socket timeout (milliseconds)")
- private Integer socketTimeout;
-
- @Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "connection ttl (milliseconds)")
- private Integer connectionTtl;
-
- @Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "whether tcp keepalive is used")
- private Boolean useTCPKeepAlive;
-
- @Override
- public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
- ResourceAllocationException, NetworkRuleConflictException {
-
- Map<String, String> dm = new HashMap<String, String>();
- dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey());
- dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey());
- dm.put(ApiConstants.S3_END_POINT, getEndPoint());
- dm.put(ApiConstants.S3_BUCKET_NAME, getBucketName());
- if (getHttpsFlag() != null) {
- dm.put(ApiConstants.S3_HTTPS_FLAG, getHttpsFlag().toString());
- }
- if (getConnectionTimeout() != null) {
- dm.put(ApiConstants.S3_CONNECTION_TIMEOUT, getConnectionTimeout().toString());
- }
- if (getMaxErrorRetry() != null) {
- dm.put(ApiConstants.S3_MAX_ERROR_RETRY, getMaxErrorRetry().toString());
- }
- if (getSocketTimeout() != null) {
- dm.put(ApiConstants.S3_SOCKET_TIMEOUT, getSocketTimeout().toString());
- }
- if (getConnectionTtl() != null) {
- dm.put(ApiConstants.S3_CONNECTION_TTL, getConnectionTtl().toString());
- }
- if (getUseTCPKeepAlive() != null) {
- dm.put(ApiConstants.S3_USE_TCP_KEEPALIVE, getUseTCPKeepAlive().toString());
- }
-
-
- try{
- ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
- ImageStoreResponse storeResponse = null;
- if (result != null) {
- storeResponse = _responseGenerator.createImageStoreResponse(result);
- storeResponse.setResponseName(getCommandName());
- storeResponse.setObjectName("secondarystorage");
- setResponseObject(storeResponse);
- } else {
- throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 secondary storage");
- }
- } catch (DiscoveryException ex) {
- s_logger.warn("Exception: ", ex);
- throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
- }
- }
-
- @Override
- public boolean equals(final Object thatObject) {
-
- if (this == thatObject) {
- return true;
- }
-
- if (thatObject == null || this.getClass() != thatObject.getClass()) {
- return false;
- }
-
- final AddS3Cmd thatAddS3Cmd = (AddS3Cmd)thatObject;
-
- if (httpsFlag != null ? !httpsFlag.equals(thatAddS3Cmd.httpsFlag) : thatAddS3Cmd.httpsFlag != null) {
- return false;
- }
-
- if (accessKey != null ? !accessKey.equals(thatAddS3Cmd.accessKey) : thatAddS3Cmd.accessKey != null) {
- return false;
- }
-
- if (connectionTimeout != null ? !connectionTimeout.equals(thatAddS3Cmd.connectionTimeout) : thatAddS3Cmd.connectionTimeout != null) {
- return false;
- }
-
- if (endPoint != null ? !endPoint.equals(thatAddS3Cmd.endPoint) : thatAddS3Cmd.endPoint != null) {
- return false;
- }
-
- if (maxErrorRetry != null ? !maxErrorRetry.equals(thatAddS3Cmd.maxErrorRetry) : thatAddS3Cmd.maxErrorRetry != null) {
- return false;
- }
-
- if (secretKey != null ? !secretKey.equals(thatAddS3Cmd.secretKey) : thatAddS3Cmd.secretKey != null) {
- return false;
- }
-
- if (socketTimeout != null ? !socketTimeout.equals(thatAddS3Cmd.socketTimeout) : thatAddS3Cmd.socketTimeout != null) {
- return false;
- }
-
- if (bucketName != null ? !bucketName.equals(thatAddS3Cmd.bucketName) : thatAddS3Cmd.bucketName != null) {
- return false;
- }
-
- if (connectionTtl != null ? !connectionTtl.equals(thatAddS3Cmd.connectionTtl) : thatAddS3Cmd.connectionTtl != null) {
- return false;
- }
-
- if (useTCPKeepAlive != null ? !useTCPKeepAlive.equals(thatAddS3Cmd.useTCPKeepAlive) : thatAddS3Cmd.useTCPKeepAlive != null) {
- return false;
- }
-
- return true;
-
- }
-
- @Override
- public int hashCode() {
-
- int result = accessKey != null ? accessKey.hashCode() : 0;
- result = 31 * result + (secretKey != null ? secretKey.hashCode() : 0);
- result = 31 * result + (endPoint != null ? endPoint.hashCode() : 0);
- result = 31 * result + (bucketName != null ? bucketName.hashCode() : 0);
- result = 31 * result + (httpsFlag != null && httpsFlag == true ? 1 : 0);
- result = 31 * result + (connectionTimeout != null ? connectionTimeout.hashCode() : 0);
- result = 31 * result + (maxErrorRetry != null ? maxErrorRetry.hashCode() : 0);
- result = 31 * result + (socketTimeout != null ? socketTimeout.hashCode() : 0);
- result = 31 * result + (connectionTtl != null ? connectionTtl.hashCode() : 0);
- result = 31 * result + (useTCPKeepAlive != null && useTCPKeepAlive == true ? 1 : 0);
-
- return result;
-
- }
-
- @Override
- public String getCommandName() {
- return s_name;
- }
-
- @Override
- public long getEntityOwnerId() {
- return ACCOUNT_ID_SYSTEM;
- }
-
- public String getAccessKey() {
- return accessKey;
- }
-
- public String getSecretKey() {
- return secretKey;
- }
-
- public String getEndPoint() {
- return endPoint;
- }
-
- public String getBucketName() {
- return bucketName;
- }
-
- public Boolean getHttpsFlag() {
- return httpsFlag;
- }
-
- public Integer getConnectionTimeout() {
- return connectionTimeout;
- }
-
- public Integer getMaxErrorRetry() {
- return maxErrorRetry;
- }
-
- public Integer getSocketTimeout() {
- return socketTimeout;
- }
-
- public Integer getConnectionTtl() {
- return connectionTtl;
- }
-
- public Boolean getUseTCPKeepAlive() {
- return useTCPKeepAlive;
- }
-}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
deleted file mode 100644
index c1889e7..0000000
--- a/api/src/org/apache/cloudstack/api/command/admin/storage/ListS3sCmd.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cloudstack.api.command.admin.storage;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.BaseListCmd;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.ImageStoreResponse;
-import org.apache.cloudstack.api.response.ListResponse;
-
-import com.cloud.exception.ConcurrentOperationException;
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.NetworkRuleConflictException;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.exception.ResourceUnavailableException;
-
-@APICommand(name = "listS3s", description = "Lists S3s", responseObject = ImageStoreResponse.class, since = "4.0.0",
- requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class ListS3sCmd extends BaseListCmd {
-
- private static final String COMMAND_NAME = "lists3sresponse";
-
- @Override
- public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
- ResourceAllocationException, NetworkRuleConflictException {
-
- ListImageStoresCmd cmd = new ListImageStoresCmd();
- cmd.setProvider("S3");
- ListResponse<ImageStoreResponse> response = _queryService.searchForImageStores(cmd);
- response.setResponseName(getCommandName());
- this.setResponseObject(response);
- }
-
- @Override
- public String getCommandName() {
- return COMMAND_NAME;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
----------------------------------------------------------------------
diff --git a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
index 659d1c5..68d53f9 100644
--- a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
+++ b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
@@ -32,7 +32,7 @@ import org.apache.cloudstack.api.response.TemplateResponse;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.template.VirtualMachineTemplate;
-@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud. ", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
+@APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud.", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd {
public static final Logger s_logger = Logger.getLogger(RegisterTemplateCmdByAdmin.class.getName());
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/client/tomcatconf/commands.properties.in
----------------------------------------------------------------------
diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in
index 4f93b97..1c788a8 100644
--- a/client/tomcatconf/commands.properties.in
+++ b/client/tomcatconf/commands.properties.in
@@ -282,12 +282,9 @@ listCapacity=3
addSwift=1
listSwifts=1
-#### s3 commands
-addS3=1
-listS3s=1
-
#### image store commands
addImageStore=1
+addImageStoreS3=1
listImageStores=1
deleteImageStore=1
createSecondaryStagingStore=1
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/client/tomcatconf/log4j-cloud.xml.in
----------------------------------------------------------------------
diff --git a/client/tomcatconf/log4j-cloud.xml.in b/client/tomcatconf/log4j-cloud.xml.in
index 587aa86..0a12e7d 100755
--- a/client/tomcatconf/log4j-cloud.xml.in
+++ b/client/tomcatconf/log4j-cloud.xml.in
@@ -162,6 +162,16 @@ under the License.
<appender-ref ref="APISERVER"/>
</logger>
+ <!-- Limit the com.amazonaws category to INFO as its DEBUG is verbose -->
+ <category name="com.amazonaws">
+ <priority value="INFO"/>
+ </category>
+
+ <!-- Limit the httpclient.wire category to INFO as its DEBUG is verbose -->
+ <category name="httpclient.wire">
+ <priority value="INFO"/>
+ </category>
+
<!-- ============================== -->
<!-- Add or remove these logger for SNMP, this logger is for SNMP alerts plugin -->
<!-- ============================== -->
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/resource/ServerResource.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/resource/ServerResource.java b/core/src/com/cloud/resource/ServerResource.java
index 1bbcbff..9030db7 100644
--- a/core/src/com/cloud/resource/ServerResource.java
+++ b/core/src/com/cloud/resource/ServerResource.java
@@ -28,7 +28,6 @@ import com.cloud.host.Host;
import com.cloud.utils.component.Manager;
/**
- *
* ServerResource is a generic container to execute commands sent
*/
public interface ServerResource extends Manager {
@@ -70,4 +69,5 @@ public interface ServerResource extends Manager {
IAgentControl getAgentControl();
void setAgentControl(IAgentControl agentControl);
+
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
index 632a809..76d1ac9 100644
--- a/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/HttpTemplateDownloader.java
@@ -47,10 +47,10 @@ import org.apache.log4j.Logger;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-import com.cloud.agent.api.storage.Proxy;
import com.cloud.storage.StorageLayer;
import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
+import com.cloud.utils.net.Proxy;
/**
* Download a template file using HTTP
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/storage/template/S3TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index ac47dec..d584cdf 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -19,303 +19,235 @@
package com.cloud.storage.template;
-import static com.cloud.utils.StringUtils.join;
-import static java.util.Arrays.asList;
-
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Date;
-
+import com.amazonaws.event.ProgressEvent;
+import com.amazonaws.event.ProgressEventType;
+import com.amazonaws.event.ProgressListener;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.StorageClass;
+import com.amazonaws.services.s3.transfer.Upload;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.utils.net.HTTPUtils;
+import com.cloud.utils.net.Proxy;
+import com.cloud.utils.storage.S3.S3Utils;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-import org.apache.commons.httpclient.ChunkedInputStream;
-import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodRetryHandler;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
-import org.apache.commons.httpclient.NoHttpResponseException;
-import org.apache.commons.httpclient.UsernamePasswordCredentials;
-import org.apache.commons.httpclient.auth.AuthScope;
+import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.ProgressEvent;
-import com.amazonaws.services.s3.model.ProgressListener;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.StorageClass;
-import com.cloud.agent.api.storage.Proxy;
-import com.cloud.agent.api.to.S3TO;
-import com.cloud.utils.Pair;
-import com.cloud.utils.S3Utils;
-import com.cloud.utils.UriUtils;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+
+import static com.cloud.utils.StringUtils.join;
+import static java.util.Arrays.asList;
/**
* Download a template file using HTTP(S)
+ *
+ * This class, once instantiated, has the purpose to download a single Template to an S3 Image Store.
+ *
+ * Execution of the instance is started when runInContext() is called.
*/
public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
- private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
- private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
-
- private String downloadUrl;
- private String installPath;
- private String s3Key;
- private String fileName;
- private String fileExtension;
- private String errorString = " ";
-
+ private static final Logger LOGGER = Logger.getLogger(S3TemplateDownloader.class.getName());
+
+ private final String downloadUrl;
+ private final String s3Key;
+ private final String fileExtension;
+ private final HttpClient httpClient;
+ private final GetMethod getMethod;
+ private final DownloadCompleteCallback downloadCompleteCallback;
+ private final S3TO s3TO;
+ private String errorString = "";
private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
private ResourceType resourceType = ResourceType.TEMPLATE;
- private final HttpClient client;
- private final HttpMethodRetryHandler myretryhandler;
- private GetMethod request;
- private DownloadCompleteCallback completionCallback;
- private S3TO s3to;
-
- private long remoteSize = 0;
- private long downloadTime = 0;
+ private long remoteSize;
+ private long downloadTime;
private long totalBytes;
private long maxTemplateSizeInByte;
private boolean resume = false;
- private boolean inited = true;
- public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
- long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
- this.s3to = s3to;
+ public S3TemplateDownloader(S3TO s3TO, String downloadUrl, String installPath, DownloadCompleteCallback downloadCompleteCallback,
+ long maxTemplateSizeInBytes, String username, String password, Proxy proxy, ResourceType resourceType) {
this.downloadUrl = downloadUrl;
- this.installPath = installPath;
- this.status = TemplateDownloader.Status.NOT_STARTED;
+ this.s3TO = s3TO;
this.resourceType = resourceType;
this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
+ this.httpClient = HTTPUtils.getHTTPClient();
+ this.downloadCompleteCallback = downloadCompleteCallback;
- this.totalBytes = 0;
- this.client = new HttpClient(s_httpClientManager);
+ // Create a GET method for the download url.
+ this.getMethod = new GetMethod(downloadUrl);
- this.myretryhandler = new HttpMethodRetryHandler() {
- @Override
- public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
- if (executionCount >= 2) {
- // Do not retry if over max retry count
- return false;
- }
- if (exception instanceof NoHttpResponseException) {
- // Retry if the server dropped connection on us
- return true;
- }
- if (!method.isRequestSent()) {
- // Retry if the request has not been sent fully or
- // if it's OK to retry methods that have been sent
- return true;
- }
- // otherwise do not retry
- return false;
- }
- };
+ // Set the retry handler, default to retry 5 times.
+ this.getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, HTTPUtils.getHttpMethodRetryHandler(5));
- try {
- request = new GetMethod(downloadUrl);
- request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
- completionCallback = callback;
-
- Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl);
- fileName = StringUtils.substringAfterLast(downloadUrl, "/");
- fileExtension = StringUtils.substringAfterLast(fileName, ".");
-
- if (proxy != null) {
- client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
- if (proxy.getUserName() != null) {
- Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword());
- client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds);
- }
- }
- if ((user != null) && (password != null)) {
- client.getParams().setAuthenticationPreemptive(true);
- Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
- client.getState().setCredentials(
- new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
- s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
- + ":" + hostAndPort.second());
- } else {
- s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
- }
- } catch (IllegalArgumentException iae) {
- errorString = iae.getMessage();
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- inited = false;
- } catch (Exception ex) {
- errorString = "Unable to start download -- check url? ";
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- s_logger.warn("Exception in constructor -- " + ex.toString());
- } catch (Throwable th) {
- s_logger.warn("throwable caught ", th);
- }
+ // Follow redirects
+ this.getMethod.setFollowRedirects(true);
+
+ // Set file extension.
+ this.fileExtension = StringUtils.substringAfterLast(StringUtils.substringAfterLast(downloadUrl, "/"), ".");
+
+ // Calculate and set S3 Key.
+ this.s3Key = join(asList(installPath, StringUtils.substringAfterLast(downloadUrl, "/")), S3Utils.SEPARATOR);
+
+ // Set proxy if available.
+ HTTPUtils.setProxy(proxy, this.httpClient);
+
+ // Set credentials if available.
+ HTTPUtils.setCredentials(username, password, this.httpClient);
}
@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
- switch (status) {
- case ABORTED:
- case UNRECOVERABLE_ERROR:
- case DOWNLOAD_FINISHED:
+ if (!status.equals(Status.NOT_STARTED)) {
+ // Only start downloading if we haven't started yet.
+ LOGGER.debug("Template download is already started, not starting again. Template: " + downloadUrl);
+
+ return 0;
+ }
+
+ int responseCode;
+ if ((responseCode = HTTPUtils.executeMethod(httpClient, getMethod)) == -1) {
+ errorString = "Exception while executing HttpMethod " + getMethod.getName() + " on URL " + downloadUrl;
+ LOGGER.warn(errorString);
+
+ status = Status.UNRECOVERABLE_ERROR;
return 0;
- default:
+ }
+
+ if (!HTTPUtils.verifyResponseCode(responseCode)) {
+ errorString = "Response code for GetMethod of " + downloadUrl + " is incorrect, responseCode: " + responseCode;
+ LOGGER.warn(errorString);
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
}
+ // Headers
+ Header contentLengthHeader = getMethod.getResponseHeader("Content-Length");
+ Header contentTypeHeader = getMethod.getResponseHeader("Content-Type");
+
+ // Check the contentLengthHeader and transferEncodingHeader.
+ if (contentLengthHeader == null) {
+ errorString = "The ContentLengthHeader of " + downloadUrl + " isn't supplied";
+ LOGGER.warn(errorString);
+
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
+ } else {
+ // The ContentLengthHeader is supplied, parse it's value.
+ remoteSize = Long.parseLong(contentLengthHeader.getValue());
+ }
+
+ if (remoteSize > maxTemplateSizeInByte) {
+ errorString = "Remote size is too large for template " + downloadUrl + " remote size is " + remoteSize + " max allowed is " + maxTemplateSizeInByte;
+ LOGGER.warn(errorString);
+
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
+ }
+
+ InputStream inputStream;
+
try {
- // execute get method
- int responseCode = HttpStatus.SC_OK;
- if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
- return 0; // FIXME: retry?
- }
- // get the total size of file
- Header contentLengthHeader = request.getResponseHeader("Content-Length");
- boolean chunked = false;
- long remoteSize2 = 0;
- if (contentLengthHeader == null) {
- Header chunkedHeader = request.getResponseHeader("Transfer-Encoding");
- if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = " Failed to receive length of download ";
- return 0; // FIXME: what status do we put here? Do we retry?
- } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) {
- chunked = true;
- }
- } else {
- remoteSize2 = Long.parseLong(contentLengthHeader.getValue());
- }
+ inputStream = new BufferedInputStream(getMethod.getResponseBodyAsStream());
+ } catch (IOException e) {
+ errorString = "Exception occurred while opening InputStream for template " + downloadUrl;
+ LOGGER.warn(errorString);
- if (remoteSize == 0) {
- remoteSize = remoteSize2;
- }
+ status = Status.UNRECOVERABLE_ERROR;
+ return 0;
+ }
- if (remoteSize > maxTemplateSizeInByte) {
- s_logger.info("Remote size is too large: " + remoteSize + " , max=" + maxTemplateSizeInByte);
- status = Status.UNRECOVERABLE_ERROR;
- errorString = "Download file size is too large";
- return 0;
- }
+ LOGGER.info("Starting download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " and size " + remoteSize + " bytes");
- if (remoteSize == 0) {
- remoteSize = maxTemplateSizeInByte;
- }
+ // Time the upload starts.
+ final Date start = new Date();
- // get content type
- String contentType = null;
- Header contentTypeHeader = request.getResponseHeader("Content-Type");
- if (contentTypeHeader != null) {
- contentType = contentTypeHeader.getValue();
- }
+ ObjectMetadata objectMetadata = new ObjectMetadata();
+ objectMetadata.setContentLength(remoteSize);
- InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
- : new ChunkedInputStream(request.getResponseBodyAsStream());
+ if (contentTypeHeader.getValue() != null) {
+ objectMetadata.setContentType(contentTypeHeader.getValue());
+ }
- s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
- + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
+ // Create the PutObjectRequest.
+ PutObjectRequest putObjectRequest = new PutObjectRequest(s3TO.getBucketName(), s3Key, inputStream, objectMetadata);
- Date start = new Date();
- // compute s3 key
- s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR);
+ // If reduced redundancy is enabled, set it.
+ if (s3TO.getEnableRRS()) {
+ putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
+ }
- // download using S3 API
- ObjectMetadata metadata = new ObjectMetadata();
- metadata.setContentLength(remoteSize);
- if (contentType != null) {
- metadata.setContentType(contentType);
- }
- PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
- // check if RRS is enabled
- if (s3to.getEnableRRS()) {
- putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
- }
- // register progress listenser
- putObjectRequest.setProgressListener(new ProgressListener() {
- @Override
- public void progressChanged(ProgressEvent progressEvent) {
- // s_logger.debug(progressEvent.getBytesTransfered()
- // + " number of byte transferd "
- // + new Date());
- totalBytes += progressEvent.getBytesTransfered();
- if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) {
- s_logger.info("download completed");
- status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
- } else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- } else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE) {
- status = TemplateDownloader.Status.ABORTED;
- } else {
- status = TemplateDownloader.Status.IN_PROGRESS;
- }
- }
+ Upload upload = S3Utils.putObject(s3TO, putObjectRequest);
+
+ upload.addProgressListener(new ProgressListener() {
+ @Override
+ public void progressChanged(ProgressEvent progressEvent) {
+
+ // Record the amount of bytes transferred.
+ totalBytes += progressEvent.getBytesTransferred();
+
+ LOGGER.trace("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + ((new Date().getTime() - start.getTime()) / 1000) + " seconds");
- });
-
- if (!s3to.getSingleUpload(remoteSize)) {
- // use TransferManager to do multipart upload
- S3Utils.mputObject(s3to, putObjectRequest);
- } else {
- // single part upload, with 5GB limit in Amazon
- S3Utils.putObject(s3to, putObjectRequest);
- while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
- && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
- && status != TemplateDownloader.Status.ABORTED) {
- // wait for completion
+ if (progressEvent.getEventType() == ProgressEventType.TRANSFER_STARTED_EVENT) {
+ status = Status.IN_PROGRESS;
+ } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
+ status = Status.DOWNLOAD_FINISHED;
+ } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_CANCELED_EVENT) {
+ status = Status.ABORTED;
+ } else if (progressEvent.getEventType() == ProgressEventType.TRANSFER_FAILED_EVENT) {
+ status = Status.UNRECOVERABLE_ERROR;
}
}
+ });
- // finished or aborted
- Date finish = new Date();
- String downloaded = "(incomplete download)";
- if (totalBytes >= remoteSize) {
- status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
- downloaded = "(download complete remote=" + remoteSize + "bytes)";
- } else {
- errorString = "Downloaded " + totalBytes + " bytes " + downloaded;
- }
- downloadTime += finish.getTime() - start.getTime();
- return totalBytes;
- } catch (HttpException hte) {
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = hte.getMessage();
- } catch (IOException ioe) {
- // probably a file write error
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = ioe.getMessage();
- } catch (AmazonClientException ex) {
- // S3 api exception
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = ex.getMessage();
+ try {
+ // Wait for the upload to complete.
+ upload.waitForCompletion();
} catch (InterruptedException e) {
- // S3 upload api exception
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- errorString = e.getMessage();
- } finally {
- // close input stream
- request.releaseConnection();
- if (callback != null) {
- callback.downloadComplete(status);
- }
+ // Interruption while waiting for the upload to complete.
+ LOGGER.warn("Interruption occurred while waiting for upload of " + downloadUrl + " to complete");
}
- return 0;
+
+ downloadTime = new Date().getTime() - start.getTime();
+
+ if (status == Status.DOWNLOAD_FINISHED) {
+ LOGGER.info("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed successfully!");
+ } else {
+ LOGGER.warn("Template download from " + downloadUrl + " to S3 bucket " + s3TO.getBucketName() + " transferred " + totalBytes + " in " + (downloadTime / 1000) + " seconds, completed with status " + status.toString());
+ }
+
+ // Close input stream
+ getMethod.releaseConnection();
+
+ // Call the callback!
+ if (callback != null) {
+ callback.downloadComplete(status);
+ }
+
+ return totalBytes;
}
public String getDownloadUrl() {
- return downloadUrl;
+ try {
+ return getMethod.getURI().toString();
+ } catch (URIException e) {
+ return null;
+ }
}
@Override
- public TemplateDownloader.Status getStatus() {
+ public Status getStatus() {
return status;
}
@@ -342,47 +274,38 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return null;
}
- return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
+ return S3Utils.getObjectStream(s3TO, s3TO.getBucketName(), s3Key);
}
public void cleanupAfterError() {
- if (status != Status.UNRECOVERABLE_ERROR) {
- s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
- return;
- }
-
- s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
+ LOGGER.warn("Cleanup after error, trying to remove object: " + s3Key);
- S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+ S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
}
@Override
- @SuppressWarnings("fallthrough")
public boolean stopDownload() {
- switch (getStatus()) {
- case IN_PROGRESS:
- if (request != null) {
- request.abort();
- }
- status = TemplateDownloader.Status.ABORTED;
- return true;
- case UNKNOWN:
- case NOT_STARTED:
- case RECOVERABLE_ERROR:
- case UNRECOVERABLE_ERROR:
- case ABORTED:
- status = TemplateDownloader.Status.ABORTED;
- case DOWNLOAD_FINISHED:
- try {
- S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
- } catch (Exception ex) {
- // ignore delete exception if it is not there
- }
- return true;
-
- default:
- return true;
+ switch (status) {
+ case IN_PROGRESS:
+ if (getMethod != null) {
+ getMethod.abort();
+ }
+ break;
+ case UNKNOWN:
+ case NOT_STARTED:
+ case RECOVERABLE_ERROR:
+ case UNRECOVERABLE_ERROR:
+ case ABORTED:
+ case DOWNLOAD_FINISHED:
+ // Remove the object if it already has been uploaded.
+ S3Utils.deleteObject(s3TO, s3TO.getBucketName(), s3Key);
+ break;
+ default:
+ break;
}
+
+ status = TemplateDownloader.Status.ABORTED;
+ return true;
}
@Override
@@ -396,18 +319,12 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
@Override
protected void runInContext() {
- try {
- download(resume, completionCallback);
- } catch (Throwable t) {
- s_logger.warn("Caught exception during download " + t.getMessage(), t);
- errorString = "Failed to install: " + t.getMessage();
- status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
- }
-
+ // Start the download!
+ download(resume, downloadCompleteCallback);
}
@Override
- public void setStatus(TemplateDownloader.Status status) {
+ public void setStatus(Status status) {
this.status = status;
}
@@ -442,7 +359,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
@Override
public boolean isInited() {
- return inited;
+ return true;
}
public ResourceType getResourceType() {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/com/cloud/storage/template/TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/TemplateDownloader.java b/core/src/com/cloud/storage/template/TemplateDownloader.java
index da9787d..5db3d24 100644
--- a/core/src/com/cloud/storage/template/TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/TemplateDownloader.java
@@ -25,16 +25,16 @@ public interface TemplateDownloader extends Runnable {
* Callback used to notify completion of download
*
*/
- public interface DownloadCompleteCallback {
+ interface DownloadCompleteCallback {
void downloadComplete(Status status);
}
- public static enum Status {
+ enum Status {
UNKNOWN, NOT_STARTED, IN_PROGRESS, ABORTED, UNRECOVERABLE_ERROR, RECOVERABLE_ERROR, DOWNLOAD_FINISHED, POST_DOWNLOAD_FINISHED
}
- public static long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
+ long DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES = 50L * 1024L * 1024L * 1024L;
/**
* Initiate download, resuming a previous one if required
@@ -42,55 +42,54 @@ public interface TemplateDownloader extends Runnable {
* @param callback completion callback to be called after download is complete
* @return bytes downloaded
*/
- public long download(boolean resume, DownloadCompleteCallback callback);
+ long download(boolean resume, DownloadCompleteCallback callback);
/**
* @return
*/
- public boolean stopDownload();
+ boolean stopDownload();
/**
* @return percent of file downloaded
*/
- public int getDownloadPercent();
+ int getDownloadPercent();
/**
* Get the status of the download
* @return status of download
*/
- public TemplateDownloader.Status getStatus();
+ TemplateDownloader.Status getStatus();
/**
* Get time taken to download so far
* @return time in seconds taken to download
*/
- public long getDownloadTime();
+ long getDownloadTime();
/**
* Get bytes downloaded
* @return bytes downloaded so far
*/
- public long getDownloadedBytes();
+ long getDownloadedBytes();
/**
* Get the error if any
* @return error string if any
*/
- public String getDownloadError();
+ String getDownloadError();
/** Get local path of the downloaded file
* @return local path of the file downloaded
*/
- public String getDownloadLocalPath();
+ String getDownloadLocalPath();
- public void setStatus(TemplateDownloader.Status status);
+ void setStatus(TemplateDownloader.Status status);
- public void setDownloadError(String string);
+ void setDownloadError(String string);
- public void setResume(boolean resume);
+ void setResume(boolean resume);
- public boolean isInited();
-
- public long getMaxTemplateSizeInBytes();
+ boolean isInited();
+ long getMaxTemplateSizeInBytes();
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
----------------------------------------------------------------------
diff --git a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
index 5dfc22e..29d737f 100644
--- a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
+++ b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
@@ -25,7 +25,7 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO;
import com.cloud.agent.api.storage.AbstractDownloadCommand;
import com.cloud.agent.api.storage.PasswordAuth;
-import com.cloud.agent.api.storage.Proxy;
+import com.cloud.utils.net.Proxy;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.storage.Storage.ImageFormat;
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
----------------------------------------------------------------------
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
index 7b5f8d9..3e5761e 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java
@@ -23,14 +23,14 @@ import java.util.Set;
public interface DataStoreProvider {
// constants for provider names
- static final String NFS_IMAGE = "NFS";
- static final String S3_IMAGE = "S3";
- static final String SWIFT_IMAGE = "Swift";
- static final String SAMPLE_IMAGE = "Sample";
- static final String SMB = "NFS";
- static final String DEFAULT_PRIMARY = "DefaultPrimary";
-
- static enum DataStoreProviderType {
+ String NFS_IMAGE = "NFS";
+ String S3_IMAGE = "S3";
+ String SWIFT_IMAGE = "Swift";
+ String SAMPLE_IMAGE = "Sample";
+ String SMB = "NFS";
+ String DEFAULT_PRIMARY = "DefaultPrimary";
+
+ enum DataStoreProviderType {
PRIMARY, IMAGE, ImageCache
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index 69dc7c7..9ab3595 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -196,7 +196,7 @@ public class TemplateServiceImpl implements TemplateService {
@Override
public void downloadBootstrapSysTemplate(DataStore store) {
- Set<VMTemplateVO> toBeDownloaded = new HashSet<VMTemplateVO>();
+ Set<VMTemplateVO> toBeDownloaded = new HashSet();
List<VMTemplateVO> rtngTmplts = _templateDao.listAllSystemVMTemplates();
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
index 3dc6ac4..8f081d3 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java
@@ -27,15 +27,15 @@ import com.cloud.storage.DataStoreRole;
import com.cloud.utils.fsm.NoTransitionException;
public interface ObjectInDataStoreManager {
- public DataObject create(DataObject dataObj, DataStore dataStore);
+ DataObject create(DataObject dataObj, DataStore dataStore);
- public boolean delete(DataObject dataObj);
+ boolean delete(DataObject dataObj);
- public boolean deleteIfNotReady(DataObject dataObj);
+ boolean deleteIfNotReady(DataObject dataObj);
- public DataObject get(DataObject dataObj, DataStore store);
+ DataObject get(DataObject dataObj, DataStore store);
- public boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
+ boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException;
DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role);
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/5c0366c9/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
index 380040b..0068c2d 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
@@ -47,7 +47,6 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.DownloadAnswer;
-import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataTO;
import com.cloud.alert.AlertManager;
@@ -58,6 +57,7 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.download.DownloadMonitor;
+import com.cloud.utils.net.Proxy;
public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
private static final Logger s_logger = Logger.getLogger(BaseImageStoreDriverImpl.class);