You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by we...@apache.org on 2023/09/01 06:16:10 UTC
[cloudstack] branch 4.18 updated: Fix direct download https compressed qcow2 template checker (#7932)
This is an automated email from the ASF dual-hosted git repository.
weizhou pushed a commit to branch 4.18
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.18 by this push:
new 57c61fb33cf Fix direct download https compressed qcow2 template checker (#7932)
57c61fb33cf is described below
commit 57c61fb33cf0a4587983d8b39b6aadd58a26a9b4
Author: Nicolas Vazquez <ni...@gmail.com>
AuthorDate: Fri Sep 1 03:16:03 2023 -0300
Fix direct download https compressed qcow2 template checker (#7932)
This PR fixes an issue on direct download while registering HTTPS compressed files
Fixes: #7929
---
.../download/HttpsDirectTemplateDownloader.java | 3 +-
.../resource/wrapper/LibvirtCheckUrlCommand.java | 5 +++
.../storage/template/DownloadManagerImpl.java | 2 +-
.../java/com/cloud/utils/storage/QCOW2Utils.java | 42 +++++++++++-----------
.../com/cloud/utils/storage/QCOW2UtilsTest.java | 6 ++--
5 files changed, 33 insertions(+), 25 deletions(-)
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
index 908bf22964b..2035aab2aca 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
@@ -183,7 +183,8 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
SSLContext context = getSSLContext();
urlConnection.setSSLSocketFactory(context.getSocketFactory());
urlConnection.connect();
- return QCOW2Utils.getVirtualSize(urlConnection.getInputStream());
+ boolean isCompressed = !url.endsWith("qcow2");
+ return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
} catch (IOException e) {
throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
}
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
index 939d43086f4..935bc8e113b 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
@@ -40,6 +40,11 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
if (checkResult) {
remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
+ if (remoteSize == null || remoteSize < 0) {
+ s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
+ "url %s, obtained size = %s", url, remoteSize));
+ return new CheckUrlAnswer(false, remoteSize);
+ }
}
return new CheckUrlAnswer(checkResult, remoteSize);
}
diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index e06d7da210d..f647b497f58 100644
--- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -347,7 +347,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
// and as such can be easily read.
try (InputStream inputStream = td.getS3ObjectInputStream();) {
- dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream));
+ dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream, false));
}
catch (IOException e) {
result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage();
diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
index afd4fe52e0a..32a54722aae 100644
--- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
+++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java
@@ -25,10 +25,9 @@ import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import org.apache.commons.compress.compressors.CompressorException;
-import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.log4j.Logger;
@@ -57,7 +56,10 @@ public final class QCOW2Utils {
* @param inputStream The QCOW2 object in stream format.
* @return The virtual size of the QCOW2 object.
*/
- public static long getVirtualSize(InputStream inputStream) throws IOException {
+ public static long getVirtualSize(InputStream inputStream, boolean isCompressed) throws IOException {
+ if (isCompressed) {
+ return getVirtualSizeFromInputStream(inputStream);
+ }
byte[] bytes = new byte[VIRTUALSIZE_HEADER_LENGTH];
if (inputStream.skip(VIRTUALSIZE_HEADER_LOCATION) != VIRTUALSIZE_HEADER_LOCATION) {
@@ -71,20 +73,13 @@ public final class QCOW2Utils {
return NumbersUtil.bytesToLong(bytes);
}
- public static long getVirtualSize(String urlStr) {
- InputStream inputStream = null;
-
+ private static long getVirtualSizeFromInputStream(InputStream inputStream) throws IOException {
try {
- URL url = new URL(urlStr);
- BufferedInputStream bufferedInputStream = new BufferedInputStream(url.openStream());
- inputStream = bufferedInputStream;
-
try {
- CompressorInputStream compressorInputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
- inputStream = compressorInputStream;
+ BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream);
+ inputStream = new CompressorStreamFactory().createCompressorInputStream(bufferedInputStream);
} catch (CompressorException e) {
LOGGER.warn(e.getMessage());
- inputStream = bufferedInputStream;
}
byte[] inputBytes = inputStream.readNBytes(VIRTUALSIZE_HEADER_LOCATION + VIRTUALSIZE_HEADER_LENGTH);
@@ -93,7 +88,7 @@ public final class QCOW2Utils {
inputMagicBytes.put(inputBytes, 0, MAGIC_HEADER_LENGTH);
ByteBuffer qcow2MagicBytes = ByteBuffer.allocate(MAGIC_HEADER_LENGTH);
- qcow2MagicBytes.put("QFI".getBytes(Charset.forName("UTF-8")));
+ qcow2MagicBytes.put("QFI".getBytes(StandardCharsets.UTF_8));
qcow2MagicBytes.put((byte)0xfb);
long virtualSize = 0L;
@@ -105,12 +100,6 @@ public final class QCOW2Utils {
}
return virtualSize;
- } catch (MalformedURLException e) {
- LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
- throw new IllegalArgumentException("Invalid URL: " + urlStr);
- } catch (IOException e) {
- LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
- throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
} finally {
if (inputStream != null) {
try {
@@ -121,4 +110,17 @@ public final class QCOW2Utils {
}
}
}
+
+ public static long getVirtualSize(String urlStr) {
+ try {
+ URL url = new URL(urlStr);
+ return getVirtualSizeFromInputStream(url.openStream());
+ } catch (MalformedURLException e) {
+ LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
+ throw new IllegalArgumentException("Invalid URL: " + urlStr);
+ } catch (IOException e) {
+ LOGGER.warn("Failed to validate for qcow2, error: " + e.getMessage());
+ throw new IllegalArgumentException("Failed to connect URL: " + urlStr);
+ }
+ }
}
diff --git a/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java b/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java
index 92881a63cda..0595b2f67db 100644
--- a/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java
+++ b/utils/src/test/java/com/cloud/utils/storage/QCOW2UtilsTest.java
@@ -24,7 +24,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import org.junit.Before;
import org.junit.Test;
@@ -67,7 +67,7 @@ public class QCOW2UtilsTest {
ByteBuffer byteBuffer = ByteBuffer.allocate(72);
// Magic
- byteBuffer.put("QFI".getBytes(Charset.forName("UTF-8")));
+ byteBuffer.put("QFI".getBytes(StandardCharsets.UTF_8));
byteBuffer.put((byte)0xfb);
// Version
@@ -116,6 +116,6 @@ public class QCOW2UtilsTest {
@Test
public void getVirtualSizeTest() throws IOException {
- assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream));
+ assertEquals(virtualSize.longValue(), QCOW2Utils.getVirtualSize(inputStream, false));
}
}