You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by wi...@apache.org on 2015/09/14 12:03:29 UTC
[1/4] git commit: updated refs/heads/master to d091b91
Repository: cloudstack
Updated Branches:
refs/heads/master 0b73d1936 -> d091b9189
Added QCOW2 virtual size checking for S3.
- Cleaned up S3TemplateDownloader
- Created static QCOW2 utils class.
- Reformatted some parts of DownloadManagerImpl
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/1971614e
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/1971614e
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/1971614e
Branch: refs/heads/master
Commit: 1971614e315b3dfe3833fae372fb87e1a2dd0e39
Parents: 28d18dc
Author: Boris Schrijver <bo...@pcextreme.nl>
Authored: Wed Sep 9 17:53:35 2015 +0200
Committer: Boris Schrijver <bo...@pcextreme.nl>
Committed: Thu Sep 10 16:45:48 2015 +0200
----------------------------------------------------------------------
.../storage/template/S3TemplateDownloader.java | 169 ++++++++++++-------
.../storage/template/DownloadManagerImpl.java | 87 +++++++---
.../src/main/java/com/cloud/utils/S3Utils.java | 16 ++
.../com/cloud/utils/storage/QCOW2Utils.java | 60 +++++++
4 files changed, 244 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1971614e/core/src/com/cloud/storage/template/S3TemplateDownloader.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
index ec44d8d..ac47dec 100644
--- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java
+++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java
@@ -27,6 +27,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
+import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.commons.httpclient.ChunkedInputStream;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header;
@@ -50,10 +52,6 @@ import com.amazonaws.services.s3.model.ProgressEvent;
import com.amazonaws.services.s3.model.ProgressListener;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.StorageClass;
-
-import org.apache.cloudstack.managed.context.ManagedContextRunnable;
-import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
-
import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.S3TO;
import com.cloud.utils.Pair;
@@ -61,46 +59,48 @@ import com.cloud.utils.S3Utils;
import com.cloud.utils.UriUtils;
/**
- * Download a template file using HTTP
- *
+ * Download a template file using HTTP(S)
*/
public class S3TemplateDownloader extends ManagedContextRunnable implements TemplateDownloader {
- public static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
+ private static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName());
private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
private String downloadUrl;
private String installPath;
private String s3Key;
private String fileName;
- public TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
- public String errorString = " ";
- private long remoteSize = 0;
- public long downloadTime = 0;
- public long totalBytes;
+ private String fileExtension;
+ private String errorString = " ";
+
+ private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
+ private ResourceType resourceType = ResourceType.TEMPLATE;
private final HttpClient client;
+ private final HttpMethodRetryHandler myretryhandler;
private GetMethod request;
- private boolean resume = false;
private DownloadCompleteCallback completionCallback;
- private S3TO s3;
- private boolean inited = true;
+ private S3TO s3to;
+ private long remoteSize = 0;
+ private long downloadTime = 0;
+ private long totalBytes;
private long maxTemplateSizeInByte;
- private ResourceType resourceType = ResourceType.TEMPLATE;
- private final HttpMethodRetryHandler myretryhandler;
- public S3TemplateDownloader(S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user,
- String password, Proxy proxy, ResourceType resourceType) {
- s3 = storageLayer;
+ private boolean resume = false;
+ private boolean inited = true;
+
+ public S3TemplateDownloader(S3TO s3to, String downloadUrl, String installPath, DownloadCompleteCallback callback,
+ long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) {
+ this.s3to = s3to;
this.downloadUrl = downloadUrl;
this.installPath = installPath;
- status = TemplateDownloader.Status.NOT_STARTED;
+ this.status = TemplateDownloader.Status.NOT_STARTED;
this.resourceType = resourceType;
- maxTemplateSizeInByte = maxTemplateSizeInBytes;
+ this.maxTemplateSizeInByte = maxTemplateSizeInBytes;
- totalBytes = 0;
- client = new HttpClient(s_httpClientManager);
+ this.totalBytes = 0;
+ this.client = new HttpClient(s_httpClientManager);
- myretryhandler = new HttpMethodRetryHandler() {
+ this.myretryhandler = new HttpMethodRetryHandler() {
@Override
public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
if (executionCount >= 2) {
@@ -128,6 +128,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
Pair<String, Integer> hostAndPort = UriUtils.validateUrl(downloadUrl);
fileName = StringUtils.substringAfterLast(downloadUrl, "/");
+ fileExtension = StringUtils.substringAfterLast(fileName, ".");
if (proxy != null) {
client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort());
@@ -139,8 +140,10 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
if ((user != null) && (password != null)) {
client.getParams().setAuthenticationPreemptive(true);
Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
- client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
- s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
+ client.getState().setCredentials(
+ new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
+ s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first()
+ + ":" + hostAndPort.second());
} else {
s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second());
}
@@ -160,11 +163,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
@Override
public long download(boolean resume, DownloadCompleteCallback callback) {
switch (status) {
- case ABORTED:
- case UNRECOVERABLE_ERROR:
- case DOWNLOAD_FINISHED:
- return 0;
- default:
+ case ABORTED:
+ case UNRECOVERABLE_ERROR:
+ case DOWNLOAD_FINISHED:
+ return 0;
+ default:
}
@@ -215,10 +218,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
contentType = contentTypeHeader.getValue();
}
- InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream()) : new ChunkedInputStream(request.getResponseBodyAsStream());
+ InputStream in = !chunked ? new BufferedInputStream(request.getResponseBodyAsStream())
+ : new ChunkedInputStream(request.getResponseBodyAsStream());
- s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName() + " remoteSize=" + remoteSize + " , max size=" +
- maxTemplateSizeInByte);
+ s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3to.getBucketName()
+ + " remoteSize=" + remoteSize + " , max size=" + maxTemplateSizeInByte);
Date start = new Date();
// compute s3 key
@@ -230,9 +234,9 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
if (contentType != null) {
metadata.setContentType(contentType);
}
- PutObjectRequest putObjectRequest = new PutObjectRequest(s3.getBucketName(), s3Key, in, metadata);
+ PutObjectRequest putObjectRequest = new PutObjectRequest(s3to.getBucketName(), s3Key, in, metadata);
// check if RRS is enabled
- if (s3.getEnableRRS()) {
+ if (s3to.getEnableRRS()) {
putObjectRequest = putObjectRequest.withStorageClass(StorageClass.ReducedRedundancy);
}
// register progress listenser
@@ -257,14 +261,15 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
});
- if (!s3.getSingleUpload(remoteSize)) {
+ if (!s3to.getSingleUpload(remoteSize)) {
// use TransferManager to do multipart upload
- S3Utils.mputObject(s3, putObjectRequest);
+ S3Utils.mputObject(s3to, putObjectRequest);
} else {
// single part upload, with 5GB limit in Amazon
- S3Utils.putObject(s3, putObjectRequest);
- while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
- status != TemplateDownloader.Status.ABORTED) {
+ S3Utils.putObject(s3to, putObjectRequest);
+ while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED
+ && status != TemplateDownloader.Status.UNRECOVERABLE_ERROR
+ && status != TemplateDownloader.Status.ABORTED) {
// wait for completion
}
}
@@ -324,32 +329,59 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return totalBytes;
}
+ /**
+ * Returns an InputStream only when the status is DOWNLOAD_FINISHED.
+ *
+ * The caller of this method must close the InputStream to prevent resource leaks!
+ *
+ * @return S3ObjectInputStream of the object.
+ */
+ public InputStream getS3ObjectInputStream() {
+ // Check if the download is finished
+ if (status != Status.DOWNLOAD_FINISHED) {
+ return null;
+ }
+
+ return S3Utils.getObjectStream(s3to, s3to.getBucketName(), s3Key);
+ }
+
+ public void cleanupAfterError() {
+ if (status != Status.UNRECOVERABLE_ERROR) {
+ s_logger.debug("S3Template downloader does not have state UNRECOVERABLE_ERROR, no cleanup neccesarry.");
+ return;
+ }
+
+ s_logger.info("Cleanup after UNRECOVERABLE_ERROR, trying to remove object: " + s3Key);
+
+ S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+ }
+
@Override
@SuppressWarnings("fallthrough")
public boolean stopDownload() {
switch (getStatus()) {
- case IN_PROGRESS:
- if (request != null) {
- request.abort();
- }
- status = TemplateDownloader.Status.ABORTED;
- return true;
- case UNKNOWN:
- case NOT_STARTED:
- case RECOVERABLE_ERROR:
- case UNRECOVERABLE_ERROR:
- case ABORTED:
- status = TemplateDownloader.Status.ABORTED;
- case DOWNLOAD_FINISHED:
- try {
- S3Utils.deleteObject(s3, s3.getBucketName(), s3Key);
- } catch (Exception ex) {
- // ignore delete exception if it is not there
- }
- return true;
+ case IN_PROGRESS:
+ if (request != null) {
+ request.abort();
+ }
+ status = TemplateDownloader.Status.ABORTED;
+ return true;
+ case UNKNOWN:
+ case NOT_STARTED:
+ case RECOVERABLE_ERROR:
+ case UNRECOVERABLE_ERROR:
+ case ABORTED:
+ status = TemplateDownloader.Status.ABORTED;
+ case DOWNLOAD_FINISHED:
+ try {
+ S3Utils.deleteObject(s3to, s3to.getBucketName(), s3Key);
+ } catch (Exception ex) {
+ // ignore delete exception if it is not there
+ }
+ return true;
- default:
- return true;
+ default:
+ return true;
}
}
@@ -359,7 +391,7 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return 0;
}
- return (int)(100.0 * totalBytes / remoteSize);
+ return (int) (100.0 * totalBytes / remoteSize);
}
@Override
@@ -417,4 +449,11 @@ public class S3TemplateDownloader extends ManagedContextRunnable implements Temp
return resourceType;
}
-}
+ public long getTotalBytes() {
+ return totalBytes;
+ }
+
+ public String getFileExtension() {
+ return fileExtension;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1971614e/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index 25c0887..2e4bb74 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -41,13 +41,12 @@ import java.util.concurrent.Executors;
import javax.ejb.Local;
import javax.naming.ConfigurationException;
-import org.apache.log4j.Logger;
-
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+import org.apache.log4j.Logger;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.storage.Proxy;
@@ -67,9 +66,9 @@ import com.cloud.storage.template.Processor;
import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.QCOW2Processor;
import com.cloud.storage.template.RawImageProcessor;
-import com.cloud.storage.template.TARProcessor;
import com.cloud.storage.template.S3TemplateDownloader;
import com.cloud.storage.template.ScpTemplateDownloader;
+import com.cloud.storage.template.TARProcessor;
import com.cloud.storage.template.TemplateConstants;
import com.cloud.storage.template.TemplateDownloader;
import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
@@ -83,6 +82,7 @@ import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
+import com.cloud.utils.storage.QCOW2Utils;
@Local(value = DownloadManager.class)
public class DownloadManagerImpl extends ManagerBase implements DownloadManager {
@@ -129,10 +129,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
this.tmpltName = tmpltName;
this.format = format;
this.hvm = hvm;
- description = descr;
- checksum = cksum;
+ this.description = descr;
+ this.checksum = cksum;
this.installPathPrefix = installPathPrefix;
- templatesize = 0;
+ this.templatesize = 0;
this.id = id;
this.resourceType = resourceType;
}
@@ -276,11 +276,27 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
threadPool.execute(td);
break;
case DOWNLOAD_FINISHED:
- if (!(td instanceof S3TemplateDownloader)) {
- // we currently only create template.properties for NFS by
- // running some post download script
+ if(td instanceof S3TemplateDownloader) {
+ // For S3 and Swift, which are considered "remote",
+ // as in the file cannot be accessed locally,
+ // we run the postRemoteDownload() method.
+ td.setDownloadError("Download success, starting install ");
+ String result = postRemoteDownload(jobId);
+ if (result != null) {
+ s_logger.error("Failed post download install: " + result);
+ td.setStatus(Status.UNRECOVERABLE_ERROR);
+ td.setDownloadError("Failed post download install: " + result);
+ ((S3TemplateDownloader) td).cleanupAfterError();
+ } else {
+ td.setStatus(Status.POST_DOWNLOAD_FINISHED);
+ td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
+ }
+ }
+ else {
+ // For other TemplateDownloaders where files are locally available,
+ // we run the postLocalDownload() method.
td.setDownloadError("Download success, starting install ");
- String result = postDownload(jobId);
+ String result = postLocalDownload(jobId);
if (result != null) {
s_logger.error("Failed post download script: " + result);
td.setStatus(Status.UNRECOVERABLE_ERROR);
@@ -289,17 +305,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
td.setStatus(Status.POST_DOWNLOAD_FINISHED);
td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
}
- } else {
- // for s3 and swift, we skip post download step and just set
- // status to trigger callback.
- td.setStatus(Status.POST_DOWNLOAD_FINISHED);
- // set template size for S3
- S3TemplateDownloader std = (S3TemplateDownloader)td;
- long size = std.totalBytes;
- DownloadJob dnld = jobs.get(jobId);
- dnld.setTemplatesize(size);
- dnld.setTemplatePhysicalSize(size);
- dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name.
}
dj.cleanup();
break;
@@ -339,12 +344,48 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
/**
- * Post download activity (install and cleanup). Executed in context of
+ * Post remote download activity (install and cleanup). Executed in context of the downloader thread.
+ */
+ private String postRemoteDownload(String jobId) {
+ String result = null;
+ DownloadJob dnld = jobs.get(jobId);
+ S3TemplateDownloader td = (S3TemplateDownloader)dnld.getTemplateDownloader();
+
+ if (td.getFileExtension().equalsIgnoreCase("QCOW2")) {
+ // The QCOW2 is the only format with a header,
+ // and as such can be easily read.
+
+ try {
+ InputStream inputStream = td.getS3ObjectInputStream();
+
+ dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream));
+
+ inputStream.close();
+ }
+ catch (IOException e) {
+ result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage();
+ }
+
+ }
+ else {
+ // For the other formats, both the virtual
+ // and actual file size are set the same.
+ dnld.setTemplatesize(td.getTotalBytes());
+ }
+
+ dnld.setTemplatePhysicalSize(td.getTotalBytes());
+ dnld.setTmpltPath(td.getDownloadLocalPath());
+
+ return result;
+ }
+
+ /**
+ * Post local download activity (install and cleanup). Executed in context of
* downloader thread
*
* @throws IOException
*/
- private String postDownload(String jobId) {
+ private String postLocalDownload(String jobId) {
DownloadJob dnld = jobs.get(jobId);
TemplateDownloader td = dnld.getTemplateDownloader();
String resourcePath = dnld.getInstallPathPrefix(); // path with mount
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1971614e/utils/src/main/java/com/cloud/utils/S3Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/main/java/com/cloud/utils/S3Utils.java b/utils/src/main/java/com/cloud/utils/S3Utils.java
index 6efe76b..c07db33 100644
--- a/utils/src/main/java/com/cloud/utils/S3Utils.java
+++ b/utils/src/main/java/com/cloud/utils/S3Utils.java
@@ -62,6 +62,7 @@ import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.Upload;
@@ -256,6 +257,21 @@ public final class S3Utils {
}
+ // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak.
+ public static S3ObjectInputStream getObjectStream(final ClientOptions clientOptions, final String bucketName, final String key) {
+
+ assert clientOptions != null;
+ assert !isBlank(bucketName);
+ assert !isBlank(key);
+
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(format("Get S3 object %1$s in " + "bucket %2$s", key, bucketName));
+ }
+
+ return acquireClient(clientOptions).getObject(bucketName, key).getObjectContent();
+
+ }
+
@SuppressWarnings("unchecked")
public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key, final File targetDirectory,
final FileNamingStrategy namingStrategy) {
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/1971614e/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
----------------------------------------------------------------------
diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
new file mode 100644
index 0000000..3e08bd6
--- /dev/null
+++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
@@ -0,0 +1,60 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.utils.storage;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.cloud.utils.NumbersUtil;
+
+public final class QCOW2Utils {
+ private static final int VIRTUALSIZE_HEADER_LOCATION = 24;
+ private static final int VIRTUALSIZE_HEADER_LENGTH = 8;
+
+ /**
+ * Private constructor -> This utility class cannot be instantiated.
+ */
+ private QCOW2Utils() {}
+
+ /**
+ * @return the header location of the virtual size field.
+ */
+ public static int getVirtualSizeHeaderLocation() {
+ return VIRTUALSIZE_HEADER_LOCATION;
+ }
+
+ /**
+ * @param inputStream The QCOW2 object in stream format.
+ * @return The virtual size of the QCOW2 object.
+ */
+ public static long getVirtualSize(InputStream inputStream) throws IOException {
+ byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH];
+
+ if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) {
+ throw new IOException("Unable to skip to the virtual size header");
+ }
+
+ if (inputStream.read(bytes) != VIRTUALSIZE_HEADER_LENGTH) {
+ throw new IOException("Unable to properly read the size");
+ }
+
+ return NumbersUtil.bytesToLong(bytes);
+ }
+}
\ No newline at end of file
[3/4] git commit: updated refs/heads/master to d091b91
Posted by wi...@apache.org.
Added Unit Tests for QCOW2Utils.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/aa19a6a0
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/aa19a6a0
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/aa19a6a0
Branch: refs/heads/master
Commit: aa19a6a05ab2ae17266201fd44b7c3c7f044e7bf
Parents: 4a770fc
Author: Boris Schrijver <bo...@pcextreme.nl>
Authored: Mon Sep 14 10:56:21 2015 +0200
Committer: Boris Schrijver <bo...@pcextreme.nl>
Committed: Mon Sep 14 10:56:21 2015 +0200
----------------------------------------------------------------------
.../com/cloud/utils/storage/QCOW2UtilsTest.java | 121 +++++++++++++++++++
1 file changed, 121 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/aa19a6a0/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java
----------------------------------------------------------------------
diff --git a/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java b/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java
new file mode 100644
index 0000000..92881a6
--- /dev/null
+++ b/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java
@@ -0,0 +1,121 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.utils.storage;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class QCOW2UtilsTest {
+
+ InputStream inputStream;
+
+ final Long virtualSize = 21474836480L;
+
+ /**
+ * The QCOW2 Header
+ *
+ * uint32_t magic;
+ * uint32_t version;
+ *
+ * uint64_t backing_file_offset;
+ * uint32_t backing_file_size;
+ *
+ * uint32_t cluster_bits;
+ * uint64_t size;
+ *
+ * uint32_t crypt_method;
+ *
+ * uint32_t l1_size;
+ * int64_t l1_table_offset;
+ *
+ * uint64_t refcount_table_offset;
+ * uint32_t refcount_table_clusters;
+ *
+ * uint32_t nb_snapshots;
+ * uint64_t snapshots_offset;
+ *
+ * @see https://people.gnome.org/~markmc/qcow-image-format.html
+ */
+
+ @Before
+ public void setup() {
+
+ ByteBuffer byteBuffer = ByteBuffer.allocate(72);
+
+ // Magic
+ byteBuffer.put("QFI".getBytes(Charset.forName("UTF-8")));
+ byteBuffer.put((byte)0xfb);
+
+ // Version
+ byteBuffer.putInt(2);
+
+ // Backing file offset
+ byteBuffer.putLong(0L);
+
+ // Backing file size
+ byteBuffer.putInt(0);
+
+ // Cluster bits
+ byteBuffer.putInt(0);
+
+ // Size
+ byteBuffer.putLong(virtualSize);
+
+ // Crypt method
+ byteBuffer.putInt(0);
+
+ // L1 Size
+ byteBuffer.putInt(0);
+
+ // L1 Table offset
+ byteBuffer.putLong(0L);
+
+ // Refcount table offset
+ byteBuffer.putLong(0L);
+
+ // Refcount table cluster
+ byteBuffer.putInt(0);
+
+ // NB Snapshots
+ byteBuffer.putInt(0);
+
+ // Snapshots offset
+ byteBuffer.putLong(0L);
+
+ inputStream = new ByteArrayInputStream(byteBuffer.array());
+ }
+
+ @Test
+ public void getVirtualSizeHeaderLocation() {
+ assertEquals(24, QCOW2Utils.getVirtualSizeHeaderLocation());
+ }
+
+ @Test
+ public void getVirtualSizeTest() throws IOException {
+ assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream));
+ }
+}
[4/4] git commit: updated refs/heads/master to d091b91
Posted by wi...@apache.org.
Merge pull request #795 from borisroman/CLOUDSTACK-8819
CLOUDSTACK-8819: Added QCOW2 virtual size checking for S3.- Cleaned up S3TemplateDownloader
- Created static QCOW2 utils class.
- Reformatted some parts of DownloadManagerImpl
* pr/795:
Added Unit Tests for QCOW2Utils.
Fixed Findbugs issue introduced by 1c6378ec0056e8c75990a4a0c15e99b2df162a75 PR #795.
Added QCOW2 virtual size checking for S3.
Signed-off-by: Wido den Hollander <wi...@widodh.nl>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/d091b918
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/d091b918
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/d091b918
Branch: refs/heads/master
Commit: d091b9189ccc27549ea993f8e919232fa64922a7
Parents: 0b73d19 aa19a6a
Author: Wido den Hollander <wi...@widodh.nl>
Authored: Mon Sep 14 12:02:02 2015 +0200
Committer: Wido den Hollander <wi...@widodh.nl>
Committed: Mon Sep 14 12:02:03 2015 +0200
----------------------------------------------------------------------
.../storage/template/DownloadManagerImpl.java | 3 +-
.../com/cloud/utils/storage/QCOW2UtilsTest.java | 121 +++++++++++++++++++
2 files changed, 123 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[2/4] git commit: updated refs/heads/master to d091b91
Posted by wi...@apache.org.
Fixed Findbugs issue introduced by 1c6378ec0056e8c75990a4a0c15e99b2df162a75 PR #795.
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/4a770fc6
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/4a770fc6
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/4a770fc6
Branch: refs/heads/master
Commit: 4a770fc63a77e44cecbf6615b55a10113055e042
Parents: 1971614
Author: Boris Schrijver <bo...@pcextreme.nl>
Authored: Mon Sep 14 10:56:11 2015 +0200
Committer: Boris Schrijver <bo...@pcextreme.nl>
Committed: Mon Sep 14 10:56:11 2015 +0200
----------------------------------------------------------------------
.../apache/cloudstack/storage/template/DownloadManagerImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4a770fc6/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index 2e4bb74..3bc0de7 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -78,6 +78,7 @@ import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.storage.template.VmdkProcessor;
import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.StringUtils;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
@@ -423,7 +424,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
if (extension.equals("iso")) {
templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_");
} else {
- templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes()).toString();
+ templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes(StringUtils.getPreferredCharset())).toString();
}
// run script to mv the temporary template file to the final template