You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ro...@apache.org on 2023/09/28 06:41:47 UTC
[cloudstack] branch 4.18 updated: Use direct download timeout configs for URL check (#7948)
This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch 4.18
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.18 by this push:
new 221f863939e Use direct download timeout configs for URL check (#7948)
221f863939e is described below
commit 221f863939ee039971bcfe14b97dcb6a3f2e6a0f
Author: Marcus Sorensen <ma...@apple.com>
AuthorDate: Thu Sep 28 00:41:38 2023 -0600
Use direct download timeout configs for URL check (#7948)
Signed-off-by: Marcus Sorensen <ml...@apple.com>
Co-authored-by: Marcus Sorensen <ml...@apple.com>
---
.../agent/directdownload/CheckUrlCommand.java | 18 +++++++++++++++
.../direct/download/DirectDownloadHelper.java | 27 +++++++++++++++++-----
.../download/HttpDirectTemplateDownloader.java | 4 ++--
.../download/HttpsDirectTemplateDownloader.java | 4 ++--
.../download/MetalinkDirectTemplateDownloader.java | 4 ++--
.../download/BaseDirectTemplateDownloaderTest.java | 2 +-
.../MetalinkDirectTemplateDownloaderTest.java | 3 ++-
.../resource/wrapper/LibvirtCheckUrlCommand.java | 11 ++++++---
.../cloud/template/HypervisorTemplateAdapter.java | 6 ++++-
9 files changed, 61 insertions(+), 18 deletions(-)
diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
index e8618d54209..b1b76da8211 100644
--- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
+++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
@@ -25,6 +25,9 @@ public class CheckUrlCommand extends Command {
private String format;
private String url;
+ private Integer connectTimeout;
+ private Integer connectionRequestTimeout;
+ private Integer socketTimeout;
public String getFormat() {
return format;
@@ -34,12 +37,27 @@ public class CheckUrlCommand extends Command {
return url;
}
+ public Integer getConnectTimeout() { return connectTimeout; }
+
+ public Integer getConnectionRequestTimeout() { return connectionRequestTimeout; }
+
+ public Integer getSocketTimeout() { return socketTimeout; }
+
public CheckUrlCommand(final String format,final String url) {
super();
this.format = format;
this.url = url;
}
+ public CheckUrlCommand(final String format,final String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+ super();
+ this.format = format;
+ this.url = url;
+ this.connectTimeout = connectTimeout;
+ this.socketTimeout = socketTimeout;
+ this.connectionRequestTimeout = connectionRequestTimeout;
+ }
+
@Override
public boolean executeInSequence() {
return false;
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
index 80509b19b1b..27e35b7074b 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadHelper.java
@@ -54,7 +54,7 @@ public class DirectDownloadHelper {
public static boolean checkUrlExistence(String url) {
try {
- DirectTemplateDownloader checker = getCheckerDownloader(url);
+ DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
return checker.checkUrl(url);
} catch (CloudRuntimeException e) {
LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
@@ -62,22 +62,37 @@ public class DirectDownloadHelper {
}
}
- private static DirectTemplateDownloader getCheckerDownloader(String url) {
+ public static boolean checkUrlExistence(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+ try {
+ DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
+ return checker.checkUrl(url);
+ } catch (CloudRuntimeException e) {
+ LOGGER.error(String.format("Cannot check URL %s is reachable due to: %s", url, e.getMessage()), e);
+ return false;
+ }
+ }
+
+ private static DirectTemplateDownloader getCheckerDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
if (url.toLowerCase().startsWith("https:")) {
- return new HttpsDirectTemplateDownloader(url);
+ return new HttpsDirectTemplateDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
} else if (url.toLowerCase().startsWith("http:")) {
- return new HttpDirectTemplateDownloader(url);
+ return new HttpDirectTemplateDownloader(url, connectTimeout, socketTimeout);
} else if (url.toLowerCase().startsWith("nfs:")) {
return new NfsDirectTemplateDownloader(url);
} else if (url.toLowerCase().endsWith(".metalink")) {
- return new MetalinkDirectTemplateDownloader(url);
+ return new MetalinkDirectTemplateDownloader(url, connectTimeout, socketTimeout);
} else {
throw new CloudRuntimeException(String.format("Cannot find a download checker for url: %s", url));
}
}
public static Long getFileSize(String url, String format) {
- DirectTemplateDownloader checker = getCheckerDownloader(url);
+ DirectTemplateDownloader checker = getCheckerDownloader(url, null, null, null);
+ return checker.getRemoteFileSize(url, format);
+ }
+
+ public static Long getFileSize(String url, String format, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+ DirectTemplateDownloader checker = getCheckerDownloader(url, connectTimeout, connectionRequestTimeout, socketTimeout);
return checker.getRemoteFileSize(url, format);
}
}
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
index 093f0604a44..e1b2f1fe429 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java
@@ -50,8 +50,8 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
protected GetMethod request;
protected Map<String, String> reqHeaders = new HashMap<>();
- protected HttpDirectTemplateDownloader(String url) {
- this(url, null, null, null, null, null, null, null);
+ protected HttpDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
+ this(url, null, null, null, null, connectTimeout, socketTimeout, null);
}
public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum,
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
index 2035aab2aca..1bee45c477d 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
@@ -65,8 +65,8 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl
protected CloseableHttpClient httpsClient;
private HttpUriRequest req;
- protected HttpsDirectTemplateDownloader(String url) {
- this(url, null, null, null, null, null, null, null, null);
+ protected HttpsDirectTemplateDownloader(String url, Integer connectTimeout, Integer connectionRequestTimeout, Integer socketTimeout) {
+ this(url, null, null, null, null, connectTimeout, socketTimeout, connectionRequestTimeout, null);
}
public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers,
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
index 83802064cdf..06578d8c2b2 100644
--- a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
+++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java
@@ -60,8 +60,8 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm
}
}
- protected MetalinkDirectTemplateDownloader(String url) {
- this(url, null, null, null, null, null, null, null);
+ protected MetalinkDirectTemplateDownloader(String url, Integer connectTimeout, Integer socketTimeout) {
+ this(url, null, null, null, null, connectTimeout, socketTimeout, null);
}
public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum,
diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
index e4f1d8f4412..2c7245662a2 100644
--- a/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
+++ b/core/src/test/java/org/apache/cloudstack/direct/download/BaseDirectTemplateDownloaderTest.java
@@ -56,7 +56,7 @@ public class BaseDirectTemplateDownloaderTest {
private HttpEntity httpEntity;
@InjectMocks
- protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl);
+ protected HttpsDirectTemplateDownloader httpsDownloader = new HttpsDirectTemplateDownloader(httpUrl, 1000, 1000, 1000);
@Before
public void init() throws IOException {
diff --git a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
index 9c6400bcdf4..68982fb915f 100644
--- a/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
+++ b/core/src/test/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloaderTest.java
@@ -25,7 +25,8 @@ import org.mockito.InjectMocks;
public class MetalinkDirectTemplateDownloaderTest extends BaseDirectTemplateDownloaderTest {
@InjectMocks
- protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl);
+ protected MetalinkDirectTemplateDownloader metalinkDownloader = new MetalinkDirectTemplateDownloader(httpsUrl, 1000, 1000);
+
@Test
public void testCheckUrlMetalink() {
metalinkDownloader.downloader = httpsDownloader;
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
index 935bc8e113b..5faad5633f3 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
@@ -35,11 +35,16 @@ public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, Chec
@Override
public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serverResource) {
final String url = cmd.getUrl();
- s_logger.info("Checking URL: " + url);
+ final Integer connectTimeout = cmd.getConnectTimeout();
+ final Integer connectionRequestTimeout = cmd.getConnectionRequestTimeout();
+ final Integer socketTimeout = cmd.getSocketTimeout();
+
+ s_logger.info(String.format("Checking URL: %s, with connect timeout: %d, connect request timeout: %d, socket timeout: %d", url, connectTimeout, connectionRequestTimeout, socketTimeout));
Long remoteSize = null;
- boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
+
+ boolean checkResult = DirectDownloadHelper.checkUrlExistence(url, connectTimeout, connectionRequestTimeout, socketTimeout);
if (checkResult) {
- remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
+ remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat(), connectTimeout, connectionRequestTimeout, socketTimeout);
if (remoteSize == null || remoteSize < 0) {
s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
"url %s, obtained size = %s", url, remoteSize));
diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
index f5c19ecff1d..cdd58ce030e 100644
--- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
+++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java
@@ -39,6 +39,7 @@ import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd;
import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd;
import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd;
+import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -167,7 +168,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
if (host == null) {
throw new CloudRuntimeException("Couldn't find a host to validate URL " + url);
}
- CheckUrlCommand cmd = new CheckUrlCommand(format, url);
+ Integer socketTimeout = DirectDownloadManager.DirectDownloadSocketTimeout.value();
+ Integer connectRequestTimeout = DirectDownloadManager.DirectDownloadConnectionRequestTimeout.value();
+ Integer connectTimeout = DirectDownloadManager.DirectDownloadConnectTimeout.value();
+ CheckUrlCommand cmd = new CheckUrlCommand(format, url, connectTimeout, connectRequestTimeout, socketTimeout);
s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
Answer answer = _agentMgr.easySend(host.getId(), cmd);
if (answer == null || !answer.getResult()) {