You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by sw...@apache.org on 2016/07/04 03:35:29 UTC
[1/2] git commit: updated refs/heads/master to 1f9bf93
Repository: cloudstack
Updated Branches:
refs/heads/master e4ba640a2 -> 1f9bf9394
[CLOUDSTACK-9423] Add ability to get virtual size of compressed VHDs
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/b0247b53
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/b0247b53
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/b0247b53
Branch: refs/heads/master
Commit: b0247b53f99ff97fd05d4d8528115ed6e7d497c0
Parents: 22c6b47
Author: Syed <sy...@gmail.com>
Authored: Mon Jun 27 16:11:14 2016 -0400
Committer: Syed <sy...@gmail.com>
Committed: Wed Jun 29 12:53:51 2016 -0400
----------------------------------------------------------------------
core/pom.xml | 5 ++
.../cloud/storage/template/VhdProcessor.java | 85 ++++++++++++++++---
.../storage/template/VhdProcessorTest.java | 36 +++++++-
core/test/resources/vhds/test.vhd | Bin 0 -> 3584 bytes
core/test/resources/vhds/test.vhd.bz2 | Bin 0 -> 212 bytes
core/test/resources/vhds/test.vhd.gz | Bin 0 -> 163 bytes
core/test/resources/vhds/test.vhd.zip | Bin 0 -> 341 bytes
7 files changed, 113 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b0247b53/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 60dc02f..f50f8af 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -47,6 +47,11 @@
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.10</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b0247b53/core/src/com/cloud/storage/template/VhdProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/com/cloud/storage/template/VhdProcessor.java b/core/src/com/cloud/storage/template/VhdProcessor.java
index 605d2a3..cb13d06 100644
--- a/core/src/com/cloud/storage/template/VhdProcessor.java
+++ b/core/src/com/cloud/storage/template/VhdProcessor.java
@@ -19,20 +19,24 @@
package com.cloud.storage.template;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Map;
-
-import javax.naming.ConfigurationException;
-
import com.cloud.exception.InternalErrorException;
-import org.apache.log4j.Logger;
-
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageLayer;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.component.AdapterBase;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorInputStream;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.apache.log4j.Logger;
+
+import javax.naming.ConfigurationException;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
/**
* VhdProcessor processes the downloaded template for VHD. It
@@ -45,10 +49,12 @@ public class VhdProcessor extends AdapterBase implements Processor {
private static final Logger s_logger = Logger.getLogger(VhdProcessor.class);
StorageLayer _storage;
private int vhdFooterSize = 512;
+ private int vhdCookieOffset = 8;
private int vhdFooterCreatorAppOffset = 28;
private int vhdFooterCreatorVerOffset = 32;
private int vhdFooterCurrentSizeOffset = 48;
private byte[][] citrixCreatorApp = { {0x74, 0x61, 0x70, 0x00}, {0x43, 0x54, 0x58, 0x53}}; /*"tap ", and "CTXS"*/
+ private String vhdIdentifierCookie = "conectix";
@Override
public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
@@ -93,10 +99,32 @@ public class VhdProcessor extends AdapterBase implements Processor {
protected long getTemplateVirtualSize(File file) throws IOException {
byte[] currentSize = new byte[8];
+ byte[] cookie = new byte[8];
byte[] creatorApp = new byte[4];
- try (FileInputStream strm = new FileInputStream(file)) {
- long skipped = strm.skip(file.length() - vhdFooterSize + vhdFooterCreatorAppOffset);
+
+ BufferedInputStream fileStream = new BufferedInputStream(new FileInputStream(file));
+ InputStream strm = fileStream;
+
+ boolean isCompressed = checkCompressed(file.getAbsolutePath());
+
+ if ( isCompressed ) {
+ try {
+ strm = new CompressorStreamFactory().createCompressorInputStream(fileStream);
+ } catch (CompressorException e) {
+ s_logger.info("error opening compressed VHD file " + file.getName());
+ return file.length();
+ }
+ } try {
+
+ //read the backup footer present at the top of the VHD file
+ strm.read(cookie);
+ if (! new String(cookie).equals(vhdIdentifierCookie)) {
+ strm.close();
+ return file.length();
+ }
+
+ long skipped = strm.skip(vhdFooterCreatorAppOffset - vhdCookieOffset);
if (skipped == -1) {
throw new IOException("Unexpected end-of-file");
}
@@ -104,7 +132,7 @@ public class VhdProcessor extends AdapterBase implements Processor {
if (read == -1) {
throw new IOException("Unexpected end-of-file");
}
- skipped = strm.skip(vhdFooterCurrentSizeOffset - vhdFooterCreatorVerOffset);
+ skipped = strm.skip(vhdFooterCurrentSizeOffset - vhdFooterCreatorVerOffset - vhdCookieOffset);
if (skipped == -1) {
throw new IOException("Unexpected end-of-file");
}
@@ -112,6 +140,13 @@ public class VhdProcessor extends AdapterBase implements Processor {
if (read == -1) {
throw new IOException("Unexpected end-of-file");
}
+ } catch (IOException e) {
+ s_logger.warn("Error reading virtual size from VHD file " + e.getMessage() + " VHD: " + file.getName());
+ return file.length();
+ } finally {
+ if (strm != null) {
+ strm.close();
+ }
}
return NumbersUtil.bytesToLong(currentSize);
@@ -128,4 +163,30 @@ public class VhdProcessor extends AdapterBase implements Processor {
return true;
}
+ private boolean checkCompressed(String fileName) throws IOException {
+
+ FileInputStream fin = null;
+ BufferedInputStream bin = null;
+ CompressorInputStream cin = null;
+
+ try {
+ fin = new FileInputStream(fileName);
+ bin = new BufferedInputStream(fin);
+ cin = new CompressorStreamFactory().createCompressorInputStream(bin);
+
+ } catch (CompressorException e) {
+ s_logger.warn(e.getMessage());
+ return false;
+
+ } catch (FileNotFoundException e) {
+ s_logger.warn(e.getMessage());
+ return false;
+ } finally {
+ if (cin != null)
+ cin.close();
+ else if (bin != null)
+ bin.close();
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b0247b53/core/test/com/cloud/storage/template/VhdProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/test/com/cloud/storage/template/VhdProcessorTest.java b/core/test/com/cloud/storage/template/VhdProcessorTest.java
index b8b362a..3c695d7 100644
--- a/core/test/com/cloud/storage/template/VhdProcessorTest.java
+++ b/core/test/com/cloud/storage/template/VhdProcessorTest.java
@@ -32,6 +32,8 @@ import org.mockito.runners.MockitoJUnitRunner;
import java.io.File;
import java.io.IOException;
+import java.net.URLDecoder;
+import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
@@ -107,4 +109,36 @@ public class VhdProcessorTest {
Assert.assertEquals(virtualSize, processor.getVirtualSize(mockFile));
Mockito.verify(mockFile,Mockito.times(0)).length();
}
-}
\ No newline at end of file
+
+ @Test
+ public void testVhdGetVirtualSize() throws Exception {
+ String vhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd").getFile(), Charset.defaultCharset().name());
+ long expectedVirtualSizeBytes = 104857600;
+ long actualVirtualSizeBytes = processor.getVirtualSize(new File(vhdPath));
+ Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+ }
+
+ @Test
+ public void testGzipVhdGetVirtualSize() throws Exception {
+ String gzipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.gz").getFile(), Charset.defaultCharset().name());
+ long expectedVirtualSizeBytes = 104857600;
+ long actualVirtualSizeBytes = processor.getVirtualSize(new File(gzipVhdPath));
+ Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+ }
+
+ @Test
+ public void testBzip2VhdGetVirtualSize() throws Exception {
+ String bzipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.bz2").getFile(), Charset.defaultCharset().name());
+ long expectedVirtualSizeBytes = 104857600;
+ long actualVirtualSizeBytes = processor.getVirtualSize(new File(bzipVhdPath));
+ Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+ }
+
+ @Test
+ public void testZipVhdGetVirtualSize() throws Exception {
+ String zipVhdPath = URLDecoder.decode(getClass().getResource("/vhds/test.vhd.zip").getFile(), Charset.defaultCharset().name());
+ long expectedVirtualSizeBytes = 341; //Zip is not a supported format, virtual size should return the filesize as a fallback
+ long actualVirtualSizeBytes = processor.getVirtualSize(new File(zipVhdPath));
+ Assert.assertEquals(expectedVirtualSizeBytes, actualVirtualSizeBytes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b0247b53/core/test/resources/vhds/test.vhd
----------------------------------------------------------------------
diff --git a/core/test/resources/vhds/test.vhd b/core/test/resources/vhds/test.vhd
new file mode 100644
index 0000000..87d5f27
Binary files /dev/null and b/core/test/resources/vhds/test.vhd differ
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b0247b53/core/test/resources/vhds/test.vhd.bz2
----------------------------------------------------------------------
diff --git a/core/test/resources/vhds/test.vhd.bz2 b/core/test/resources/vhds/test.vhd.bz2
new file mode 100644
index 0000000..02821b63
Binary files /dev/null and b/core/test/resources/vhds/test.vhd.bz2 differ
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b0247b53/core/test/resources/vhds/test.vhd.gz
----------------------------------------------------------------------
diff --git a/core/test/resources/vhds/test.vhd.gz b/core/test/resources/vhds/test.vhd.gz
new file mode 100644
index 0000000..e48bf2b
Binary files /dev/null and b/core/test/resources/vhds/test.vhd.gz differ
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/b0247b53/core/test/resources/vhds/test.vhd.zip
----------------------------------------------------------------------
diff --git a/core/test/resources/vhds/test.vhd.zip b/core/test/resources/vhds/test.vhd.zip
new file mode 100644
index 0000000..4857d5a
Binary files /dev/null and b/core/test/resources/vhds/test.vhd.zip differ
[2/2] git commit: updated refs/heads/master to 1f9bf93
Posted by sw...@apache.org.
Merge pull request #1598 from syed/vhd-compressed-size
[CLOUDSTACK-9423] Add ability to get virtual size of compressed VHDsWith object store like Swift as secondary storage, if a compressed VHD is uploaded as a template, the `VHDProcessor` incorrectly calculates the virutal size leading to the template being useless. This fix tries to guess the virtual size by partially decompressing it and falls back to a sensible default which is the size of the file.
Before the fix: template.properties on Swift
```
uniquename=routing-1
filename=routing-1.vhd
size=263417314
virtualsize=2894447637315205059
```
After the fix
```
uniquename=routing-1
filename=routing-1.vhd
size=263417314
virtualsize=3145728000
```
Look at the `virutalsize` in both cases
* pr/1598:
[CLOUDSTACK-9423] Add ability to get virtual size of compressed VHDs
Signed-off-by: Will Stevens <wi...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/1f9bf939
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/1f9bf939
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/1f9bf939
Branch: refs/heads/master
Commit: 1f9bf93948b37f3132f55ea6ce7041cae57696da
Parents: e4ba640 b0247b5
Author: Will Stevens <wi...@gmail.com>
Authored: Sun Jul 3 23:34:55 2016 -0400
Committer: Will Stevens <wi...@gmail.com>
Committed: Sun Jul 3 23:34:56 2016 -0400
----------------------------------------------------------------------
core/pom.xml | 5 ++
.../cloud/storage/template/VhdProcessor.java | 85 ++++++++++++++++---
.../storage/template/VhdProcessorTest.java | 36 +++++++-
core/test/resources/vhds/test.vhd | Bin 0 -> 3584 bytes
core/test/resources/vhds/test.vhd.bz2 | Bin 0 -> 212 bytes
core/test/resources/vhds/test.vhd.gz | Bin 0 -> 163 bytes
core/test/resources/vhds/test.vhd.zip | Bin 0 -> 341 bytes
7 files changed, 113 insertions(+), 13 deletions(-)
----------------------------------------------------------------------