You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by GitBox <gi...@apache.org> on 2018/01/09 06:52:27 UTC
[GitHub] rhtyd closed pull request #2379: CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates
rhtyd closed pull request #2379: CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates
URL: https://github.com/apache/cloudstack/pull/2379
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java
index 7e802205f5c..d2669c03aeb 100644
--- a/agent/src/com/cloud/agent/Agent.java
+++ b/agent/src/com/cloud/agent/Agent.java
@@ -37,6 +37,7 @@
import javax.naming.ConfigurationException;
+import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
import org.apache.cloudstack.ca.SetupCertificateAnswer;
import org.apache.cloudstack.ca.SetupCertificateCommand;
import org.apache.cloudstack.ca.SetupKeyStoreCommand;
@@ -551,6 +552,8 @@ protected void processRequest(final Request request, final Link link) {
answer = setupAgentKeystore((SetupKeyStoreCommand) cmd);
} else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) {
answer = setupAgentCertificate((SetupCertificateCommand) cmd);
+ } else if (cmd instanceof SetupDirectDownloadCertificate) {
+ answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd);
} else {
if (cmd instanceof ReadyCommand) {
processReadyCommand(cmd);
@@ -600,6 +603,31 @@ protected void processRequest(final Request request, final Link link) {
}
}
+ private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd) {
+ String certificate = cmd.getCertificate();
+ String certificateName = cmd.getCertificateName();
+ s_logger.info("Importing certificate " + certificateName + " into keystore");
+
+ final File agentFile = PropertiesUtil.findConfigFile("agent.properties");
+ if (agentFile == null) {
+ return new Answer(cmd, false, "Failed to find agent.properties file");
+ }
+
+ final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.defaultKeystoreFile;
+
+ String cerFile = agentFile.getParent() + "/" + certificateName + ".cer";
+ Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile));
+
+ String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
+ String privatePasswordCmd = String.format(privatePasswordFormat, agentFile.getAbsolutePath());
+ String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
+
+ String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt";
+ String importCmd = String.format(importCommandFormat, cerFile, keyStoreFile, certificateName, privatePassword);
+ Script.runSimpleBashScript(importCmd);
+ return new Answer(cmd, true, "Certificate " + certificateName + " imported");
+ }
+
public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) {
final String keyStorePassword = cmd.getKeystorePassword();
final long validityDays = cmd.getValidityDays();
diff --git a/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloader.java
new file mode 100644
index 00000000000..a88b4526e9d
--- /dev/null
+++ b/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloader.java
@@ -0,0 +1,71 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.direct.download;
+
+public interface DirectTemplateDownloader {
+
+ class DirectTemplateInformation {
+ private String installPath;
+ private Long size;
+ private String checksum;
+
+ public DirectTemplateInformation(String installPath, Long size, String checksum) {
+ this.installPath = installPath;
+ this.size = size;
+ this.checksum = checksum;
+ }
+
+ public String getInstallPath() {
+ return installPath;
+ }
+
+ public Long getSize() {
+ return size;
+ }
+
+ public String getChecksum() {
+ return checksum;
+ }
+ }
+
+ /**
+ * Perform template download to pool specified on downloader creation
+ * @return true if successful, false if not
+ */
+ boolean downloadTemplate();
+
+ /**
+ * Perform extraction (if necessary) and installation of previously downloaded template
+ * @return true if successful, false if not
+ */
+ boolean extractAndInstallDownloadedTemplate();
+
+ /**
+ * Get template information after it is properly installed on pool
+ * @return template information
+ */
+ DirectTemplateInformation getTemplateInformation();
+
+ /**
+ * Perform checksum validation of previously downloadeed template
+ * @return true if successful, false if not
+ */
+ boolean validateChecksum();
+}
diff --git a/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java b/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java
new file mode 100644
index 00000000000..3b6bc9a6dd3
--- /dev/null
+++ b/agent/src/com/cloud/agent/direct/download/DirectTemplateDownloaderImpl.java
@@ -0,0 +1,185 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import org.apache.cloudstack.utils.security.ChecksumValue;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.File;
+import java.util.UUID;
+
+public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader {
+
+ private String url;
+ private String destPoolPath;
+ private Long templateId;
+ private String downloadedFilePath;
+ private String installPath;
+ private String checksum;
+
+ protected DirectTemplateDownloaderImpl(final String url, final String destPoolPath, final Long templateId, final String checksum) {
+ this.url = url;
+ this.destPoolPath = destPoolPath;
+ this.templateId = templateId;
+ this.checksum = checksum;
+ }
+
+ private static String directDownloadDir = "template";
+
+ /**
+ * Return direct download temporary path to download template
+ */
+ protected static String getDirectDownloadTempPath(Long templateId) {
+ String templateIdAsString = String.valueOf(templateId);
+ return directDownloadDir + File.separator + templateIdAsString.substring(0,1) +
+ File.separator + templateIdAsString;
+ }
+
+ /**
+ * Create folder on path if it does not exist
+ */
+ protected void createFolder(String path) {
+ File dir = new File(path);
+ if (!dir.exists()) {
+ dir.mkdirs();
+ }
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public String getDestPoolPath() {
+ return destPoolPath;
+ }
+
+ public Long getTemplateId() {
+ return templateId;
+ }
+
+ public String getDownloadedFilePath() {
+ return downloadedFilePath;
+ }
+
+ public void setDownloadedFilePath(String filePath) {
+ this.downloadedFilePath = filePath;
+ }
+
+ /**
+ * Return filename from url
+ */
+ public String getFileNameFromUrl() {
+ String[] urlParts = url.split("/");
+ return urlParts[urlParts.length - 1];
+ }
+
+ /**
+ * Checks if downloaded template is extractable
+ * @return true if it should be extracted, false if not
+ */
+ private boolean isTemplateExtractable() {
+ String type = Script.runSimpleBashScript("file " + downloadedFilePath + " | awk -F' ' '{print $2}'");
+ return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip");
+ }
+
+ @Override
+ public boolean extractAndInstallDownloadedTemplate() {
+ installPath = UUID.randomUUID().toString();
+ if (isTemplateExtractable()) {
+ extractDownloadedTemplate();
+ } else {
+ Script.runSimpleBashScript("mv " + downloadedFilePath + " " + getInstallFullPath());
+ }
+ return true;
+ }
+
+ /**
+ * Return install full path
+ */
+ private String getInstallFullPath() {
+ return destPoolPath + File.separator + installPath;
+ }
+
+ /**
+ * Return extract command to execute given downloaded file
+ */
+ private String getExtractCommandForDownloadedFile() {
+ if (downloadedFilePath.endsWith(".zip")) {
+ return "unzip -p " + downloadedFilePath + " | cat > " + getInstallFullPath();
+ } else if (downloadedFilePath.endsWith(".bz2")) {
+ return "bunzip2 -c " + downloadedFilePath + " > " + getInstallFullPath();
+ } else if (downloadedFilePath.endsWith(".gz")) {
+ return "gunzip -c " + downloadedFilePath + " > " + getInstallFullPath();
+ } else {
+ throw new CloudRuntimeException("Unable to extract template " + templateId + " on " + downloadedFilePath);
+ }
+ }
+
+ /**
+ * Extract downloaded template into installPath, remove compressed file
+ */
+ private void extractDownloadedTemplate() {
+ String extractCommand = getExtractCommandForDownloadedFile();
+ Script.runSimpleBashScript(extractCommand);
+ Script.runSimpleBashScript("rm -f " + downloadedFilePath);
+ }
+
+ @Override
+ public DirectTemplateInformation getTemplateInformation() {
+ String sizeResult = Script.runSimpleBashScript("ls -als " + getInstallFullPath() + " | awk '{print $1}'");
+ long size = Long.parseLong(sizeResult);
+ return new DirectTemplateInformation(installPath, size, checksum);
+ }
+
+ /**
+ * Return checksum command from algorithm
+ */
+ private String getChecksumCommandFromAlgorithm(String algorithm) {
+ if (algorithm.equalsIgnoreCase("MD5")) {
+ return "md5sum";
+ } else if (algorithm.equalsIgnoreCase("SHA-1")) {
+ return "sha1sum";
+ } else if (algorithm.equalsIgnoreCase("SHA-224")) {
+ return "sha224sum";
+ } else if (algorithm.equalsIgnoreCase("SHA-256")) {
+ return "sha256sum";
+ } else if (algorithm.equalsIgnoreCase("SHA-384")) {
+ return "sha384sum";
+ } else if (algorithm.equalsIgnoreCase("SHA-512")) {
+ return "sha512sum";
+ } else {
+ throw new CloudRuntimeException("Unknown checksum algorithm: " + algorithm);
+ }
+ }
+
+ @Override
+ public boolean validateChecksum() {
+ if (StringUtils.isNotBlank(checksum)) {
+ ChecksumValue providedChecksum = new ChecksumValue(checksum);
+ String algorithm = providedChecksum.getAlgorithm();
+ String checksumCommand = "echo '%s %s' | %s -c --quiet";
+ String cmd = String.format(checksumCommand, providedChecksum.getChecksum(), downloadedFilePath, getChecksumCommandFromAlgorithm(algorithm));
+ int result = Script.runSimpleBashScriptForExitValue(cmd);
+ return result == 0;
+ }
+ return true;
+ }
+}
diff --git a/agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java
new file mode 100644
index 00000000000..1f36e43ff12
--- /dev/null
+++ b/agent/src/com/cloud/agent/direct/download/HttpDirectTemplateDownloader.java
@@ -0,0 +1,122 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.HttpMethodRetryHandler;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.NoHttpResponseException;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.IOException;
+import java.util.Map;
+
+public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
+
+ private HttpClient client;
+ private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager();
+ private static final int CHUNK_SIZE = 1024 * 1024; //1M
+ protected HttpMethodRetryHandler myretryhandler;
+ public static final Logger s_logger = Logger.getLogger(HttpDirectTemplateDownloader.class.getName());
+ protected GetMethod request;
+
+ public HttpDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum, Map<String, String> headers) {
+ super(url, destPoolPath, templateId, checksum);
+ client = new HttpClient(s_httpClientManager);
+ myretryhandler = createRetryTwiceHandler();
+ request = createRequest(url, headers);
+ String downloadDir = getDirectDownloadTempPath(templateId);
+ createTemporaryDirectoryAndFile(downloadDir);
+ }
+
+ protected void createTemporaryDirectoryAndFile(String downloadDir) {
+ createFolder(getDestPoolPath() + File.separator + downloadDir);
+ File f = new File(getDestPoolPath() + File.separator + downloadDir + File.separator + getFileNameFromUrl());
+ setDownloadedFilePath(f.getAbsolutePath());
+ }
+
+ protected GetMethod createRequest(String downloadUrl, Map<String, String> headers) {
+ GetMethod request = new GetMethod(downloadUrl);
+ request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler);
+ request.setFollowRedirects(true);
+ if (MapUtils.isNotEmpty(headers)) {
+ for (String key : headers.keySet()) {
+ request.setRequestHeader(key, headers.get(key));
+ }
+ }
+ return request;
+ }
+
+ protected HttpMethodRetryHandler createRetryTwiceHandler() {
+ return new HttpMethodRetryHandler() {
+ @Override
+ public boolean retryMethod(final HttpMethod method, final IOException exception, int executionCount) {
+ if (executionCount >= 2) {
+ // Do not retry if over max retry count
+ return false;
+ }
+ if (exception instanceof NoHttpResponseException) {
+ // Retry if the server dropped connection on us
+ return true;
+ }
+ if (!method.isRequestSent()) {
+ // Retry if the request has not been sent fully or
+ // if it's OK to retry methods that have been sent
+ return true;
+ }
+ // otherwise do not retry
+ return false;
+ }
+ };
+ }
+
+ @Override
+ public boolean downloadTemplate() {
+ try {
+ client.executeMethod(request);
+ } catch (IOException e) {
+ throw new CloudRuntimeException("Error on HTTP request: " + e.getMessage());
+ }
+ return performDownload();
+ }
+
+ protected boolean performDownload() {
+ try (
+ InputStream in = request.getResponseBodyAsStream();
+ OutputStream out = new FileOutputStream(getDownloadedFilePath());
+ ) {
+ IOUtils.copy(in, out);
+ } catch (IOException e) {
+ s_logger.error("Error downloading template " + getTemplateId() + " due to: " + e.getMessage());
+ return false;
+ }
+ return true;
+ }
+}
\ No newline at end of file
diff --git a/agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java
new file mode 100644
index 00000000000..436303f71cf
--- /dev/null
+++ b/agent/src/com/cloud/agent/direct/download/HttpsDirectTemplateDownloader.java
@@ -0,0 +1,89 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.ssl.SSLContexts;
+
+import javax.net.ssl.SSLContext;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+
+public class HttpsDirectTemplateDownloader extends HttpDirectTemplateDownloader {
+
+ private CloseableHttpClient httpsClient;
+ private HttpUriRequest req;
+
+ public HttpsDirectTemplateDownloader(String url, Long templateId, String destPoolPath, String checksum) {
+ super(url, templateId, destPoolPath, checksum, null);
+ SSLContext sslcontext = null;
+ try {
+ sslcontext = getSSLContext();
+ } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) {
+ throw new CloudRuntimeException("Failure getting SSL context for HTTPS downloader: " + e.getMessage());
+ }
+ SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslcontext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ httpsClient = HttpClients.custom().setSSLSocketFactory(factory).build();
+ req = createUriRequest(url);
+ }
+
+ protected HttpUriRequest createUriRequest(String downloadUrl) {
+ return new HttpGet(downloadUrl);
+ }
+
+ private SSLContext getSSLContext() throws KeyStoreException, NoSuchAlgorithmException, CertificateException, IOException, KeyManagementException {
+ KeyStore trustStore = KeyStore.getInstance("jks");
+ FileInputStream instream = new FileInputStream(new File("/etc/cloudstack/agent/cloud.jks"));
+ try {
+ String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null";
+ String privatePasswordCmd = String.format(privatePasswordFormat, "/etc/cloudstack/agent/agent.properties");
+ String privatePassword = Script.runSimpleBashScript(privatePasswordCmd);
+ trustStore.load(instream, privatePassword.toCharArray());
+ } finally {
+ instream.close();
+ }
+ return SSLContexts.custom()
+ .loadTrustMaterial(trustStore, new TrustSelfSignedStrategy())
+ .build();
+ }
+
+ @Override
+ public boolean downloadTemplate() {
+ try {
+ httpsClient.execute(req);
+ } catch (IOException e) {
+ throw new CloudRuntimeException("Error on HTTPS request: " + e.getMessage());
+ }
+ return performDownload();
+ }
+}
\ No newline at end of file
diff --git a/agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java
new file mode 100644
index 00000000000..e4ecd6d9c5c
--- /dev/null
+++ b/agent/src/com/cloud/agent/direct/download/MetalinkDirectTemplateDownloader.java
@@ -0,0 +1,49 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.script.Script;
+
+import java.io.File;
+
+public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
+
+ private String downloadDir;
+
+ public MetalinkDirectTemplateDownloader(String url, String destPoolPath, Long templateId, String checksum) {
+ super(url, destPoolPath, templateId, checksum);
+ String relativeDir = getDirectDownloadTempPath(templateId);
+ downloadDir = getDestPoolPath() + File.separator + relativeDir;
+ createFolder(downloadDir);
+ }
+
+ @Override
+ public boolean downloadTemplate() {
+ String downloadCommand = "aria2c " + getUrl() + " -d " + downloadDir + " --check-integrity=true";
+ Script.runSimpleBashScript(downloadCommand);
+ //Remove .metalink file
+ Script.runSimpleBashScript("rm -f " + downloadDir + File.separator + getFileNameFromUrl());
+ String fileName = Script.runSimpleBashScript("ls " + downloadDir);
+ if (fileName == null) {
+ return false;
+ }
+ setDownloadedFilePath(downloadDir + File.separator + fileName);
+ return true;
+ }
+}
diff --git a/agent/src/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java b/agent/src/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java
new file mode 100644
index 00000000000..16901afedf1
--- /dev/null
+++ b/agent/src/com/cloud/agent/direct/download/NfsDirectTemplateDownloader.java
@@ -0,0 +1,70 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import com.cloud.utils.UriUtils;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.UUID;
+
+public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
+
+ private String srcHost;
+ private String srcPath;
+
+ private static final String mountCommand = "mount -t nfs %s %s";
+
+ /**
+ * Parse url and set srcHost and srcPath
+ */
+ private void parseUrl() {
+ URI uri = null;
+ String url = getUrl();
+ try {
+ uri = new URI(UriUtils.encodeURIComponent(url));
+ if (uri.getScheme() != null && uri.getScheme().equalsIgnoreCase("nfs")) {
+ srcHost = uri.getHost();
+ srcPath = uri.getPath();
+ }
+ } catch (URISyntaxException e) {
+ throw new CloudRuntimeException("Invalid NFS url " + url + " caused error: " + e.getMessage());
+ }
+ }
+
+ public NfsDirectTemplateDownloader(String url, String destPool, Long templateId, String checksum) {
+ super(url, destPool, templateId, checksum);
+ parseUrl();
+ }
+
+ @Override
+ public boolean downloadTemplate() {
+ String mountSrcUuid = UUID.randomUUID().toString();
+ String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid);
+ Script.runSimpleBashScript(mount);
+ String downloadDir = getDestPoolPath() + File.separator + getDirectDownloadTempPath(getTemplateId());
+ setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl());
+ Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath());
+ Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid);
+ return true;
+ }
+}
diff --git a/agent/test/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java b/agent/test/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java
new file mode 100644
index 00000000000..b244d02f499
--- /dev/null
+++ b/agent/test/com/cloud/agent/direct/download/DirectTemplateDownloaderImplTest.java
@@ -0,0 +1,36 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.agent.direct.download;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DirectTemplateDownloaderImplTest {
+
+ private static final Long templateId = 202l;
+
+ @Test
+ public void testGetDirectDownloadTempPath() {
+ String path = DirectTemplateDownloaderImpl.getDirectDownloadTempPath(templateId);
+ Assert.assertEquals("template/2/202", path);
+ }
+}
diff --git a/api/pom.xml b/api/pom.xml
index ce90eea4548..5daa76a3982 100644
--- a/api/pom.xml
+++ b/api/pom.xml
@@ -61,6 +61,11 @@
<artifactId>commons-lang3</artifactId>
<version>${cs.commons-lang3.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-framework-direct-download</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/api/src/com/cloud/storage/VMTemplateStorageResourceAssoc.java b/api/src/com/cloud/storage/VMTemplateStorageResourceAssoc.java
index d40eafe7a15..f43d5331222 100644
--- a/api/src/com/cloud/storage/VMTemplateStorageResourceAssoc.java
+++ b/api/src/com/cloud/storage/VMTemplateStorageResourceAssoc.java
@@ -22,7 +22,7 @@
public interface VMTemplateStorageResourceAssoc extends InternalIdentity {
public static enum Status {
- UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED
+ UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED
}
String getInstallPath();
diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java
index 20a44b07df3..cc7780045cd 100644
--- a/api/src/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/org/apache/cloudstack/api/ApiConstants.java
@@ -83,6 +83,7 @@
public static final String DESTINATION_ZONE_ID = "destzoneid";
public static final String DETAILS = "details";
public static final String DEVICE_ID = "deviceid";
+ public static final String DIRECT_DOWNLOAD = "directdownload";
public static final String DISK_OFFERING_ID = "diskofferingid";
public static final String DISK_SIZE = "disksize";
public static final String UTILIZATION = "utilization";
diff --git a/api/src/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java b/api/src/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java
new file mode 100755
index 00000000000..cd95e10d608
--- /dev/null
+++ b/api/src/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.direct.download;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.exception.NetworkRuleConflictException;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.direct.download.DirectDownloadManager;
+import org.apache.log4j.Logger;
+
+import javax.inject.Inject;
+
+@APICommand(name = UploadTemplateDirectDownloadCertificate.APINAME,
+ description = "Upload a certificate for HTTPS direct template download on KVM hosts",
+ responseObject = SuccessResponse.class,
+ requestHasSensitiveInfo = true,
+ responseHasSensitiveInfo = true,
+ since = "4.11.0",
+ authorized = {RoleType.Admin})
+public class UploadTemplateDirectDownloadCertificate extends BaseCmd {
+
+ @Inject
+ DirectDownloadManager directDownloadManager;
+
+ private static final Logger LOG = Logger.getLogger(UploadTemplateDirectDownloadCertificate.class);
+ public static final String APINAME = "uploadTemplateDirectDownloadCertificate";
+
+ @Parameter(name = ApiConstants.CERTIFICATE, type = BaseCmd.CommandType.STRING, required = true, length = 65535,
+ description = "SSL certificate")
+ private String certificate;
+
+ @Parameter(name = ApiConstants.NAME , type = BaseCmd.CommandType.STRING, required = true,
+ description = "Name for the uploaded certificate")
+ private String name;
+
+ @Parameter(name = ApiConstants.HYPERVISOR, type = BaseCmd.CommandType.STRING, required = true, description = "Hypervisor type")
+ private String hypervisor;
+
+ @Override
+ public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+ if (!hypervisor.equalsIgnoreCase("kvm")) {
+ throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Currently supporting KVM hosts only");
+ }
+
+ try {
+ directDownloadManager.uploadCertificateToHosts(certificate, name);;
+ setResponseObject(new SuccessResponse(getCommandName()));
+ } catch (Exception e) {
+ throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
+ }
+ }
+
+ @Override
+ public String getCommandName() {
+ return UploadTemplateDirectDownloadCertificate.APINAME;
+ }
+
+ @Override
+ public long getEntityOwnerId() {
+ return CallContext.current().getCallingAccount().getId();
+ }
+}
+
+
diff --git a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
index 7281197305b..ba4772b6697 100644
--- a/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
+++ b/api/src/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java
@@ -40,17 +40,7 @@
@Override
public void execute() throws ResourceAllocationException{
try {
- if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
- throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
- "Both zoneid and zoneids cannot be specified at the same time");
-
- if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
- throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
- "Either zoneid or zoneids is required. Both cannot be null.");
-
- if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
- throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
- "Parameter zoneids cannot combine all zones (-1) option with other zones");
+ validateParameters();
VirtualMachineTemplate template = _templateService.registerTemplate(this);
if (template != null){
diff --git a/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java
index 3112287fb9c..745b87dd941 100644
--- a/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java
+++ b/api/src/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java
@@ -108,6 +108,11 @@
description = "true if ISO contains XS/VMWare tools inorder to support dynamic scaling of VM CPU/memory")
protected Boolean isDynamicallyScalable;
+ @Parameter(name=ApiConstants.DIRECT_DOWNLOAD,
+ type = CommandType.BOOLEAN,
+ description = "true if ISO should bypass Secondary Storage and be downloaded to Primary Storage on deployment")
+ private Boolean directDownload;
+
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@@ -168,6 +173,10 @@ public Boolean isDynamicallyScalable() {
return isDynamicallyScalable == null ? Boolean.FALSE : isDynamicallyScalable;
}
+ public boolean isDirectDownload() {
+ return directDownload == null ? false : directDownload;
+ }
+
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
diff --git a/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java
index 2bd7b2d38d1..333b363d16a 100644
--- a/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java
+++ b/api/src/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java
@@ -16,6 +16,7 @@
// under the License.
package org.apache.cloudstack.api.command.user.template;
+import com.cloud.hypervisor.Hypervisor;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
@@ -155,6 +156,11 @@
"zone template and will be copied to all zones. ")
protected List<Long> zoneIds;
+ @Parameter(name=ApiConstants.DIRECT_DOWNLOAD,
+ type = CommandType.BOOLEAN,
+ description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment")
+ private Boolean directDownload;
+
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@@ -263,6 +269,10 @@ public Boolean isRoutingType() {
return isRoutingType;
}
+ public boolean isDirectDownload() {
+ return directDownload == null ? false : directDownload;
+ }
+
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@@ -289,17 +299,7 @@ public long getEntityOwnerId() {
@Override
public void execute() throws ResourceAllocationException {
try {
- if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
- throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
- "Both zoneid and zoneids cannot be specified at the same time");
-
- if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
- throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
- "Either zoneid or zoneids is required. Both cannot be null.");
-
- if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
- throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
- "Parameter zoneids cannot combine all zones (-1) option with other zones");
+ validateParameters();
VirtualMachineTemplate template = _templateService.registerTemplate(this);
if (template != null) {
@@ -317,4 +317,23 @@ public void execute() throws ResourceAllocationException {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, ex1.getMessage());
}
}
+
+ protected void validateParameters() {
+ if ((zoneId != null) && (zoneIds != null && !zoneIds.isEmpty()))
+ throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+ "Both zoneid and zoneids cannot be specified at the same time");
+
+ if (zoneId == null && (zoneIds == null || zoneIds.isEmpty()))
+ throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+ "Either zoneid or zoneids is required. Both cannot be null.");
+
+ if (zoneIds != null && zoneIds.size() > 1 && zoneIds.contains(-1L))
+ throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+ "Parameter zoneids cannot combine all zones (-1) option with other zones");
+
+ if (isDirectDownload() && !getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString())) {
+ throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+ "Parameter directdownload is only allowed for KVM templates");
+ }
+ }
}
diff --git a/api/src/org/apache/cloudstack/api/response/TemplateResponse.java b/api/src/org/apache/cloudstack/api/response/TemplateResponse.java
index 7cbcd1dc9ff..ab2b8a12628 100644
--- a/api/src/org/apache/cloudstack/api/response/TemplateResponse.java
+++ b/api/src/org/apache/cloudstack/api/response/TemplateResponse.java
@@ -185,6 +185,10 @@
@Param(description = "true if template contains XS/VMWare tools inorder to support dynamic scaling of VM cpu/memory")
private Boolean isDynamicallyScalable;
+ @SerializedName(ApiConstants.DIRECT_DOWNLOAD)
+ @Param(description = "KVM Only: true if template is directly downloaded to Primary Storage bypassing Secondary Storage")
+ private Boolean directDownload;
+
public TemplateResponse() {
tags = new LinkedHashSet<ResourceTagResponse>();
}
@@ -362,4 +366,12 @@ public String getZoneId() {
public void setBits(int bits) {
this.bits = bits;
}
+
+ public void setDirectDownload(Boolean directDownload) {
+ this.directDownload = directDownload;
+ }
+
+ public Boolean getDirectDownload() {
+ return directDownload;
+ }
}
diff --git a/api/src/org/apache/cloudstack/direct/download/DirectDownloadManager.java b/api/src/org/apache/cloudstack/direct/download/DirectDownloadManager.java
new file mode 100644
index 00000000000..b3f0841a6e8
--- /dev/null
+++ b/api/src/org/apache/cloudstack/direct/download/DirectDownloadManager.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.utils.component.PluggableService;
+import org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService;
+
+public interface DirectDownloadManager extends DirectDownloadService, PluggableService {
+
+}
diff --git a/client/pom.xml b/client/pom.xml
index 96cbae420aa..cfc1f873826 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -318,6 +318,11 @@
<artifactId>cloud-framework-ca</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.cloudstack</groupId>
+ <artifactId>cloud-framework-direct-download</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.cloudstack</groupId>
<artifactId>cloud-framework-ipc</artifactId>
diff --git a/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml b/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
index feca134a197..4ec917e3419 100644
--- a/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
+++ b/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml
@@ -319,4 +319,8 @@
class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
</bean>
+ <bean id="directDownloadRegistry"
+ class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
+ </bean>
+
</beans>
diff --git a/core/resources/META-INF/cloudstack/direct-download/module.properties b/core/resources/META-INF/cloudstack/direct-download/module.properties
new file mode 100644
index 00000000000..63e1a8b604f
--- /dev/null
+++ b/core/resources/META-INF/cloudstack/direct-download/module.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name=direct-download
+parent=backend
\ No newline at end of file
diff --git a/core/resources/META-INF/cloudstack/direct-download/spring-lifecycle-direct-download-context-inheritable.xml b/core/resources/META-INF/cloudstack/direct-download/spring-lifecycle-direct-download-context-inheritable.xml
new file mode 100644
index 00000000000..052a86b4811
--- /dev/null
+++ b/core/resources/META-INF/cloudstack/direct-download/spring-lifecycle-direct-download-context-inheritable.xml
@@ -0,0 +1,32 @@
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-3.0.xsd"
+>
+
+ <bean class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
+ <property name="registry" ref="directDownloadRegistry" />
+ <property name="typeClass" value="org.apache.cloudstack.framework.agent.direct.download.DirectDownloadService" />
+ </bean>
+
+</beans>
diff --git a/core/src/com/cloud/storage/resource/StorageProcessor.java b/core/src/com/cloud/storage/resource/StorageProcessor.java
index e5832cc23d4..5d57616b7a4 100644
--- a/core/src/com/cloud/storage/resource/StorageProcessor.java
+++ b/core/src/com/cloud/storage/resource/StorageProcessor.java
@@ -19,6 +19,7 @@
package com.cloud.storage.resource;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.CreateObjectCommand;
@@ -71,4 +72,6 @@
public Answer snapshotAndCopy(SnapshotAndCopyCommand cmd);
public Answer resignature(ResignatureCommand cmd);
+
+ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd);
}
diff --git a/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java
index d9d2993dbaf..8c0399e9e19 100644
--- a/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java
+++ b/core/src/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java
@@ -19,6 +19,7 @@
package com.cloud.storage.resource;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.log4j.Logger;
import org.apache.cloudstack.storage.command.AttachCommand;
@@ -66,7 +67,9 @@ public Answer handleStorageCommands(StorageSubSystemCommand command) {
} else if (command instanceof SnapshotAndCopyCommand) {
return processor.snapshotAndCopy((SnapshotAndCopyCommand)command);
} else if (command instanceof ResignatureCommand) {
- return processor.resignature((ResignatureCommand)command);
+ return processor.resignature((ResignatureCommand) command);
+ } else if (command instanceof DirectDownloadCommand) {
+ return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command);
}
return new Answer((Command)command, false, "not implemented yet");
diff --git a/core/src/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/com/cloud/storage/template/MetalinkTemplateDownloader.java
new file mode 100644
index 00000000000..149c6a1e826
--- /dev/null
+++ b/core/src/com/cloud/storage/template/MetalinkTemplateDownloader.java
@@ -0,0 +1,66 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.storage.template;
+
+import com.cloud.storage.StorageLayer;
+import com.cloud.utils.script.Script;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+
+public class MetalinkTemplateDownloader extends TemplateDownloaderBase implements TemplateDownloader {
+
+ private TemplateDownloader.Status status = TemplateDownloader.Status.NOT_STARTED;
+
+ private static final Logger LOGGER = Logger.getLogger(MetalinkTemplateDownloader.class.getName());
+
+ public MetalinkTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSize) {
+ super(storageLayer, downloadUrl, toDir, maxTemplateSize, callback);
+ String[] parts = _downloadUrl.split("/");
+ String filename = parts[parts.length - 1];
+ _toFile = toDir + File.separator + filename;
+ }
+
+ @Override
+ public long download(boolean resume, DownloadCompleteCallback callback) {
+ if (!status.equals(Status.NOT_STARTED)) {
+ // Only start downloading if we haven't started yet.
+ LOGGER.debug("Template download is already started, not starting again. Template: " + _downloadUrl);
+ return 0;
+ }
+ status = Status.IN_PROGRESS;
+ Script.runSimpleBashScript("aria2c " + _downloadUrl + " -d " + _toDir);
+ status = Status.DOWNLOAD_FINISHED;
+ String sizeResult = Script.runSimpleBashScript("ls -als " + _toFile + " | awk '{print $1}'");
+ long size = Long.parseLong(sizeResult);
+ return size;
+ }
+
+ @Override
+ public int getDownloadPercent() {
+ if (status == Status.DOWNLOAD_FINISHED) {
+ return 100;
+ } else if (status == Status.IN_PROGRESS) {
+ return 50;
+ } else {
+ return 0;
+ }
+ }
+
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/CheckUrlAnswer.java b/core/src/org/apache/cloudstack/agent/directdownload/CheckUrlAnswer.java
new file mode 100644
index 00000000000..41fd7cda2e4
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/CheckUrlAnswer.java
@@ -0,0 +1,37 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Answer;
+
+public class CheckUrlAnswer extends Answer {
+
+ private Long templateSize;
+
+ public CheckUrlAnswer(final boolean result, final Long templateSize) {
+ super(null);
+ this.result = result;
+ this.templateSize = templateSize;
+ }
+
+ public Long getTemplateSize() {
+ return templateSize;
+ }
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
new file mode 100644
index 00000000000..ed499974f5a
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/CheckUrlCommand.java
@@ -0,0 +1,42 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Command;
+
+public class CheckUrlCommand extends Command {
+
+ private String url;
+
+ public String getUrl() {
+ return url;
+ }
+
+ public CheckUrlCommand(final String url) {
+ super();
+ this.url = url;
+ }
+
+ @Override
+ public boolean executeInSequence() {
+ return false;
+ }
+
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadAnswer.java b/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadAnswer.java
new file mode 100644
index 00000000000..e4e559d8102
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadAnswer.java
@@ -0,0 +1,48 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Answer;
+
+public class DirectDownloadAnswer extends Answer {
+
+ private Long templateSize;
+ private String installPath;
+
+ public DirectDownloadAnswer(final boolean result, final String msg) {
+ super(null);
+ this.result = result;
+ this.details = msg;
+ }
+
+ public DirectDownloadAnswer(final boolean result, final Long size, final String installPath) {
+ super(null);
+ this.result = result;
+ this.templateSize = size;
+ this.installPath = installPath;
+ }
+
+ public long getTemplateSize() {
+ return templateSize;
+ }
+
+ public String getInstallPath() {
+ return installPath;
+ }
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
new file mode 100644
index 00000000000..140ad99bb5e
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+public abstract class DirectDownloadCommand extends StorageSubSystemCommand {
+
+ public enum DownloadProtocol {
+ HTTP, HTTPS, NFS, METALINK
+ }
+
+ private String url;
+ private Long templateId;
+ private PrimaryDataStoreTO destPool;
+ private String checksum;
+
+ protected DirectDownloadCommand (final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
+ this.url = url;
+ this.templateId = templateId;
+ this.destPool = destPool;
+ this.checksum = checksum;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public Long getTemplateId() {
+ return templateId;
+ }
+
+ public PrimaryDataStoreTO getDestPool() {
+ return destPool;
+ }
+
+ public String getChecksum() {
+ return checksum;
+ }
+
+ @Override
+ public void setExecuteInSequence(boolean inSeq) {
+ }
+
+ @Override
+ public boolean executeInSequence() {
+ return false;
+ }
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
new file mode 100644
index 00000000000..525c8bf50fb
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/HttpDirectDownloadCommand.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+import java.util.Map;
+
+public class HttpDirectDownloadCommand extends DirectDownloadCommand {
+
+ private Map<String, String> headers;
+
+ public HttpDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
+ super(url, templateId, destPool, checksum);
+ this.headers = headers;
+ }
+
+ public Map<String, String> getHeaders() {
+ return headers;
+ }
+
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
new file mode 100644
index 00000000000..26ed59e41cb
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/HttpsDirectDownloadCommand.java
@@ -0,0 +1,31 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+import java.util.Map;
+
+public class HttpsDirectDownloadCommand extends DirectDownloadCommand {
+
+ public HttpsDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum, Map<String, String> headers) {
+ super(url, templateId, destPool, checksum);
+ }
+}
\ No newline at end of file
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
new file mode 100644
index 00000000000..92ec7453425
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/MetalinkDirectDownloadCommand.java
@@ -0,0 +1,29 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+public class MetalinkDirectDownloadCommand extends DirectDownloadCommand {
+
+ public MetalinkDirectDownloadCommand(String url, Long templateId, PrimaryDataStoreTO destPool, String checksum) {
+ super(url, templateId, destPool, checksum);
+ }
+
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java b/core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
new file mode 100644
index 00000000000..55cef2a2f6a
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/NfsDirectDownloadCommand.java
@@ -0,0 +1,29 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+
+public class NfsDirectDownloadCommand extends DirectDownloadCommand {
+
+ public NfsDirectDownloadCommand(final String url, final Long templateId, final PrimaryDataStoreTO destPool, final String checksum) {
+ super(url, templateId, destPool, checksum);
+ }
+
+}
diff --git a/core/src/org/apache/cloudstack/agent/directdownload/SetupDirectDownloadCertificate.java b/core/src/org/apache/cloudstack/agent/directdownload/SetupDirectDownloadCertificate.java
new file mode 100644
index 00000000000..836b321f227
--- /dev/null
+++ b/core/src/org/apache/cloudstack/agent/directdownload/SetupDirectDownloadCertificate.java
@@ -0,0 +1,45 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.agent.directdownload;
+
+import com.cloud.agent.api.Command;
+
+public class SetupDirectDownloadCertificate extends Command {
+
+ private String certificate;
+ private String certificateName;
+
+ public SetupDirectDownloadCertificate(String certificate, String name) {
+ this.certificate = certificate;
+ this.certificateName = name;
+ }
+
+ public String getCertificate() {
+ return certificate;
+ }
+
+ public String getCertificateName() {
+ return certificateName;
+ }
+
+ @Override
+ public boolean executeInSequence() {
+ return false;
+ }
+}
diff --git a/debian/control b/debian/control
index 009ed20a9bc..42d3e1a5fab 100644
--- a/debian/control
+++ b/debian/control
@@ -22,7 +22,7 @@ Description: CloudStack server library
Package: cloudstack-agent
Architecture: all
-Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 1.2.2), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release, init-system-helpers (>= 1.14~)
+Depends: ${python:Depends}, openjdk-8-jre-headless | java8-runtime-headless | java8-runtime, cloudstack-common (= ${source:Version}), lsb-base (>= 4.0), libcommons-daemon-java, openssh-client, qemu-kvm (>= 1.0), libvirt-bin (>= 1.2.2), uuid-runtime, iproute, ebtables, vlan, jsvc, ipset, python-libvirt, ethtool, iptables, lsb-release, init-system-helpers (>= 1.14~), aria2
Recommends: init-system-helpers
Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
Description: CloudStack agent
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
index fd22da12e48..b213625efad 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
@@ -36,4 +36,8 @@
TemplateInfo getReadyTemplateOnCache(long templateId);
List<TemplateInfo> listTemplateOnCache(long templateId);
+
+ TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId);
+
+ boolean isTemplateMarkedForDirectDownload(long templateId);
}
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
index 50b0524ee98..0f7cc6f9de5 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java
@@ -25,4 +25,6 @@
String getUniqueName();
String getInstallPath();
+
+ boolean isDirectDownload();
}
diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
index 75a7ad96c4b..04aefbec31f 100644
--- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
+++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
@@ -108,4 +108,5 @@ public VolumeInfo getVolume() {
SnapshotInfo takeSnapshot(VolumeInfo volume);
VolumeInfo updateHypervisorSnapshotReserveForVolume(DiskOffering diskOffering, long volumeId, HypervisorType hyperType);
+
}
diff --git a/engine/components-api/src/com/cloud/resource/ResourceManager.java b/engine/components-api/src/com/cloud/resource/ResourceManager.java
index d96434b0838..1f7d3cbea54 100755
--- a/engine/components-api/src/com/cloud/resource/ResourceManager.java
+++ b/engine/components-api/src/com/cloud/resource/ResourceManager.java
@@ -195,4 +195,6 @@
* @return Details of groupNames and enabled VGPU type with remaining capacity.
*/
HashMap<String, HashMap<String, VgpuTypesInfo>> getGPUStatistics(HostVO host);
+
+ HostVO findOneRandomRunningHostByHypervisor(HypervisorType type);
}
diff --git a/engine/components-api/src/com/cloud/template/TemplateManager.java b/engine/components-api/src/com/cloud/template/TemplateManager.java
index 68c3b585692..2dc6296fc51 100644
--- a/engine/components-api/src/com/cloud/template/TemplateManager.java
+++ b/engine/components-api/src/com/cloud/template/TemplateManager.java
@@ -18,6 +18,7 @@
import java.util.List;
+import com.cloud.deploy.DeployDestination;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -119,7 +120,7 @@
List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId);
- TemplateInfo prepareIso(long isoId, long dcId);
+ TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId);
/**
@@ -127,7 +128,7 @@
*
* @param VirtualMachineProfile
*/
- void prepareIsoForVmProfile(VirtualMachineProfile profile);
+ void prepareIsoForVmProfile(VirtualMachineProfile profile, DeployDestination dest);
public static final String MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT = "Message.RegisterPublicTemplate.Event";
public static final String MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT = "Message.ResetTemplatePermission.Event";
diff --git a/engine/orchestration/pom.xml b/engine/orchestration/pom.xml
index 7d747058391..c69e29c9970 100755
--- a/engine/orchestration/pom.xml
+++ b/engine/orchestration/pom.xml
@@ -58,13 +58,6 @@
<artifactId>cloud-utils</artifactId>
<version>${project.version}</version>
</dependency>
- <!--
- <dependency>
- <groupId>org.apache.cloudstack</groupId>
- <artifactId>cloud-server</artifactId>
- <version>${project.version}</version>
- </dependency>
- -->
</dependencies>
<build>
<plugins>
diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
index 7669b3b98a9..ed1e3e06cf9 100644
--- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
+++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java
@@ -1108,7 +1108,7 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest
//if (vm.getType() == VirtualMachine.Type.User && vm.getTemplate().getFormat() == ImageFormat.ISO) {
if (vm.getType() == VirtualMachine.Type.User) {
- _tmpltMgr.prepareIsoForVmProfile(vm);
+ _tmpltMgr.prepareIsoForVmProfile(vm, dest);
//DataTO dataTO = tmplFactory.getTemplate(vm.getTemplate().getId(), DataStoreRole.Image, vm.getVirtualMachine().getDataCenterId()).getTO();
//DiskTO iso = new DiskTO(dataTO, 3L, null, Volume.Type.ISO);
//vm.addDisk(iso);
@@ -1287,9 +1287,13 @@ public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest
TemplateInfo templ = tmplFactory.getReadyTemplateOnImageStore(templateId, dest.getDataCenter().getId());
if (templ == null) {
- s_logger.debug("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
-
- throw new CloudRuntimeException("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
+ if (tmplFactory.isTemplateMarkedForDirectDownload(templateId)) {
+ // Template is marked for direct download bypassing Secondary Storage
+ templ = tmplFactory.getReadyBypassedTemplateOnPrimaryStore(templateId, destPool.getId(), dest.getHost().getId());
+ } else {
+ s_logger.debug("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
+ throw new CloudRuntimeException("can't find ready template: " + templateId + " for data center " + dest.getDataCenter().getId());
+ }
}
PrimaryDataStore primaryDataStore = (PrimaryDataStore)destPool;
diff --git a/engine/schema/resources/META-INF/db/schema-41000to41100.sql b/engine/schema/resources/META-INF/db/schema-41000to41100.sql
index 6c6655c8e46..27a2b261147 100644
--- a/engine/schema/resources/META-INF/db/schema-41000to41100.sql
+++ b/engine/schema/resources/META-INF/db/schema-41000to41100.sql
@@ -524,6 +524,111 @@ ADD COLUMN `forsystemvms` TINYINT(1) NOT NULL DEFAULT '0' COMMENT 'Indicates if
ALTER TABLE `cloud`.`op_dc_ip_address_alloc`
ADD COLUMN `vlan` INT(10) UNSIGNED NULL COMMENT 'Vlan the management network range is on';
+-- CLOUDSTACK-10146: Bypass Secondary Storage for KVM templates
+ALTER TABLE `cloud`.`vm_template`
+ADD COLUMN `direct_download` TINYINT(1) DEFAULT '0' COMMENT 'Indicates if Secondary Storage is bypassed and template is downloaded to Primary Storage';
+
+CREATE OR REPLACE VIEW `template_view` AS
+ SELECT
+ `vm_template`.`id` AS `id`,
+ `vm_template`.`uuid` AS `uuid`,
+ `vm_template`.`unique_name` AS `unique_name`,
+ `vm_template`.`name` AS `name`,
+ `vm_template`.`public` AS `public`,
+ `vm_template`.`featured` AS `featured`,
+ `vm_template`.`type` AS `type`,
+ `vm_template`.`hvm` AS `hvm`,
+ `vm_template`.`bits` AS `bits`,
+ `vm_template`.`url` AS `url`,
+ `vm_template`.`format` AS `format`,
+ `vm_template`.`created` AS `created`,
+ `vm_template`.`checksum` AS `checksum`,
+ `vm_template`.`display_text` AS `display_text`,
+ `vm_template`.`enable_password` AS `enable_password`,
+ `vm_template`.`dynamically_scalable` AS `dynamically_scalable`,
+ `vm_template`.`state` AS `template_state`,
+ `vm_template`.`guest_os_id` AS `guest_os_id`,
+ `guest_os`.`uuid` AS `guest_os_uuid`,
+ `guest_os`.`display_name` AS `guest_os_name`,
+ `vm_template`.`bootable` AS `bootable`,
+ `vm_template`.`prepopulate` AS `prepopulate`,
+ `vm_template`.`cross_zones` AS `cross_zones`,
+ `vm_template`.`hypervisor_type` AS `hypervisor_type`,
+ `vm_template`.`extractable` AS `extractable`,
+ `vm_template`.`template_tag` AS `template_tag`,
+ `vm_template`.`sort_key` AS `sort_key`,
+ `vm_template`.`removed` AS `removed`,
+ `vm_template`.`enable_sshkey` AS `enable_sshkey`,
+ `source_template`.`id` AS `source_template_id`,
+ `source_template`.`uuid` AS `source_template_uuid`,
+ `account`.`id` AS `account_id`,
+ `account`.`uuid` AS `account_uuid`,
+ `account`.`account_name` AS `account_name`,
+ `account`.`type` AS `account_type`,
+ `domain`.`id` AS `domain_id`,
+ `domain`.`uuid` AS `domain_uuid`,
+ `domain`.`name` AS `domain_name`,
+ `domain`.`path` AS `domain_path`,
+ `projects`.`id` AS `project_id`,
+ `projects`.`uuid` AS `project_uuid`,
+ `projects`.`name` AS `project_name`,
+ `data_center`.`id` AS `data_center_id`,
+ `data_center`.`uuid` AS `data_center_uuid`,
+ `data_center`.`name` AS `data_center_name`,
+ `launch_permission`.`account_id` AS `lp_account_id`,
+ `template_store_ref`.`store_id` AS `store_id`,
+ `image_store`.`scope` AS `store_scope`,
+ `template_store_ref`.`state` AS `state`,
+ `template_store_ref`.`download_state` AS `download_state`,
+ `template_store_ref`.`download_pct` AS `download_pct`,
+ `template_store_ref`.`error_str` AS `error_str`,
+ `template_store_ref`.`size` AS `size`,
+ `template_store_ref`.physical_size AS `physical_size`,
+ `template_store_ref`.`destroyed` AS `destroyed`,
+ `template_store_ref`.`created` AS `created_on_store`,
+ `vm_template_details`.`name` AS `detail_name`,
+ `vm_template_details`.`value` AS `detail_value`,
+ `resource_tags`.`id` AS `tag_id`,
+ `resource_tags`.`uuid` AS `tag_uuid`,
+ `resource_tags`.`key` AS `tag_key`,
+ `resource_tags`.`value` AS `tag_value`,
+ `resource_tags`.`domain_id` AS `tag_domain_id`,
+ `domain`.`uuid` AS `tag_domain_uuid`,
+ `domain`.`name` AS `tag_domain_name`,
+ `resource_tags`.`account_id` AS `tag_account_id`,
+ `account`.`account_name` AS `tag_account_name`,
+ `resource_tags`.`resource_id` AS `tag_resource_id`,
+ `resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
+ `resource_tags`.`resource_type` AS `tag_resource_type`,
+ `resource_tags`.`customer` AS `tag_customer`,
+ CONCAT(`vm_template`.`id`,
+ '_',
+ IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`,
+ `vm_template`.`direct_download` AS `direct_download`
+ FROM
+ ((((((((((((`vm_template`
+ JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`)))
+ JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`)))
+ JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`)))
+ LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
+ LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`)))
+ LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`)))
+ LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`)
+ AND (`template_store_ref`.`store_role` = 'Image')
+ AND (`template_store_ref`.`destroyed` = 0))))
+ LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`)
+ AND (`template_store_ref`.`store_id` IS NOT NULL)
+ AND (`image_store`.`id` = `template_store_ref`.`store_id`))))
+ LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`)
+ AND ISNULL(`template_store_ref`.`store_id`)
+ AND ISNULL(`template_zone_ref`.`removed`))))
+ LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`)
+ OR (`template_zone_ref`.`zone_id` = `data_center`.`id`))))
+ LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`)))
+ LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`)
+ AND ((`resource_tags`.`resource_type` = 'Template')
+ OR (`resource_tags`.`resource_type` = 'ISO')))));
+
-- CLOUDSTACK-10109: Enable dedication of public IPs to SSVM and CPVM
ALTER TABLE `cloud`.`user_ip_address`
ADD COLUMN `forsystemvms` TINYINT(1) NOT NULL DEFAULT '0' COMMENT 'true if IP is set to system vms, false if not';
diff --git a/engine/schema/src/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/com/cloud/storage/VMTemplateVO.java
index d28c23b1a50..e31e54d7365 100644
--- a/engine/schema/src/com/cloud/storage/VMTemplateVO.java
+++ b/engine/schema/src/com/cloud/storage/VMTemplateVO.java
@@ -146,6 +146,9 @@
@Column(name = "dynamically_scalable")
protected boolean dynamicallyScalable;
+ @Column(name = "direct_download")
+ private boolean directDownload;
+
@Override
public String getUniqueName() {
return uniqueName;
@@ -188,7 +191,7 @@ private VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic,
public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url,
boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable,
- HypervisorType hyperType, String templateTag, Map<String, String> details, boolean sshKeyEnabled, boolean isDynamicallyScalable) {
+ HypervisorType hyperType, String templateTag, Map<String, String> details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload) {
this(id,
name,
format,
@@ -212,6 +215,7 @@ public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic,
enableSshKey = sshKeyEnabled;
dynamicallyScalable = isDynamicallyScalable;
state = State.Active;
+ this.directDownload = directDownload;
}
public static VMTemplateVO createPreHostIso(Long id, String uniqueName, String name, ImageFormat format, boolean isPublic, boolean featured, TemplateType type,
@@ -605,6 +609,10 @@ public void setUpdated(Date updated) {
this.updated = updated;
}
+ public boolean isDirectDownload() {
+ return directDownload;
+ }
+
@Override
public Class<?> getEntityType() {
return VirtualMachineTemplate.class;
diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java
index 93aad153371..d2c4a99e6ae 100644
--- a/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java
+++ b/engine/schema/src/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java
@@ -23,6 +23,8 @@
import java.util.List;
+import javax.inject.Inject;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@@ -44,6 +46,9 @@
public class VMTemplatePoolDaoImpl extends GenericDaoBase<VMTemplateStoragePoolVO, Long> implements VMTemplatePoolDao {
public static final Logger s_logger = Logger.getLogger(VMTemplatePoolDaoImpl.class.getName());
+ @Inject
+ DataStoreManager dataStoreManager;
+
protected final SearchBuilder<VMTemplateStoragePoolVO> PoolSearch;
protected final SearchBuilder<VMTemplateStoragePoolVO> TemplateSearch;
protected final SearchBuilder<VMTemplateStoragePoolVO> PoolTemplateSearch;
diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
index 4da779c42a7..a6e609e7d87 100644
--- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
+++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
@@ -83,4 +83,10 @@
void expireDnldUrlsForZone(Long dcId);
List<TemplateDataStoreVO> listByTemplateState(VirtualMachineTemplate.State... states);
+
+ TemplateDataStoreVO createTemplateDirectDownloadEntry(long templateId, Long size);
+
+ TemplateDataStoreVO getReadyBypassedTemplate(long templateId);
+
+ boolean isTemplateMarkedForDirectDownload(long templateId);
}
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
index 134e74d58f6..afce5d2d27d 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
@@ -18,16 +18,23 @@
*/
package org.apache.cloudstack.storage.image;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.utils.exception.CloudRuntimeException;
+import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.store.TemplateObject;
@@ -51,6 +58,12 @@
VMTemplatePoolDao templatePoolDao;
@Inject
TemplateDataStoreDao templateStoreDao;
+ @Inject
+ DirectDownloadManager directDownloadManager;
+ @Inject
+ HostDao hostDao;
+ @Inject
+ PrimaryDataStoreDao primaryDataStoreDao;
@Override
public TemplateInfo getTemplate(long templateId, DataStore store) {
@@ -137,7 +150,6 @@ public TemplateInfo getReadyTemplateOnCache(long templateId) {
} else {
return null;
}
-
}
@Override
@@ -153,4 +165,70 @@ public TemplateInfo getReadyTemplateOnCache(long templateId) {
return tmplObjs;
}
+ /**
+ * Given existing spool refs, return one pool id existing on pools and refs
+ */
+ private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existingRefs, List<StoragePoolVO> pools) {
+ if (pools.isEmpty()) {
+ throw new CloudRuntimeException("No storage pools found");
+ }
+ if (existingRefs.isEmpty()) {
+ return pools.get(0).getId();
+ } else {
+ for (VMTemplateStoragePoolVO ref : existingRefs) {
+ for (StoragePoolVO p : pools) {
+ if (ref.getPoolId() == p.getId()) {
+ return p.getId();
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Retrieve storage pools with scope = cluster or zone matching clusterId or dataCenterId depending on their scope
+ */
+ private List<StoragePoolVO> getStoragePoolsFromClusterOrZone(Long clusterId, long dataCenterId, Hypervisor.HypervisorType hypervisorType) {
+ List<StoragePoolVO> pools = new ArrayList<>();
+ if (clusterId != null) {
+ List<StoragePoolVO> clusterPools = primaryDataStoreDao.listPoolsByCluster(clusterId);
+ pools.addAll(clusterPools);
+ }
+ List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisor(dataCenterId, hypervisorType);
+ pools.addAll(zonePools);
+ return pools;
+ }
+
+ @Override
+ public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId) {
+ VMTemplateVO templateVO = imageDataDao.findById(templateId);
+ if (templateVO == null || !templateVO.isDirectDownload()) {
+ return null;
+ }
+ Long pool = poolId;
+ if (poolId == null) {
+ //Get ISO from existing pool ref
+ HostVO host = hostDao.findById(hostId);
+ List<StoragePoolVO> pools = getStoragePoolsFromClusterOrZone(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType());
+ List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateId);
+ pool = getOneMatchingPoolIdFromRefs(existingRefs, pools);
+ }
+ if (pool == null) {
+ throw new CloudRuntimeException("No storage pool found where to download template: " + templateId);
+ }
+ VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId);
+ if (spoolRef == null) {
+ directDownloadManager.downloadTemplate(templateId, pool, hostId);
+ }
+ DataStore store = storeMgr.getDataStore(pool, DataStoreRole.Primary);
+ return this.getTemplate(templateId, store);
+ }
+
+ @Override
+ public boolean isTemplateMarkedForDirectDownload(long templateId) {
+ VMTemplateVO templateVO = imageDataDao.findById(templateId);
+ return templateVO.isDirectDownload();
+ }
+
}
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
index 7c1695eebea..52191e19ade 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
@@ -445,6 +445,9 @@ public void handleTemplateSync(DataStore store) {
} catch (NoTransitionException e) {
s_logger.error("Unexpected state transition exception for template " + tmplt.getName() + ". Details: " + e.getMessage());
}
+ } else if (tmplt.isDirectDownload()) {
+ s_logger.info("Template " + tmplt.getName() + ":" + tmplt.getId() + " is marked for direct download, discarding it for download on image stores");
+ toBeDownloaded.remove(tmplt);
} else {
s_logger.info("Template Sync did not find " + uniqueName + " on image store " + storeId + ", may request download based on available hypervisor types");
if (tmpltStore != null) {
diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
index 6e78f190d5d..db3cf6540d6 100644
--- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
+++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
@@ -344,6 +344,14 @@ public String getInstallPath() {
return obj != null ? obj.getInstallPath() : null;
}
+ @Override
+ public boolean isDirectDownload() {
+ if (this.imageVO == null) {
+ return false;
+ }
+ return this.imageVO.isDirectDownload();
+ }
+
public void setInstallPath(String installPath) {
this.installPath = installPath;
}
diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java
index aa23a43291c..f64037619a8 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java
@@ -22,12 +22,12 @@
import javax.inject.Inject;
-import org.springframework.stereotype.Component;
-
-import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
-import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
+import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
+import org.springframework.stereotype.Component;
+
import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager;
import com.cloud.storage.DataStoreRole;
diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
index 066503b0208..2372e8444cc 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
@@ -26,6 +26,7 @@
import javax.inject.Inject;
import javax.naming.ConfigurationException;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
@@ -67,6 +68,7 @@
private SearchBuilder<TemplateDataStoreVO> storeTemplateDownloadStatusSearch;
private SearchBuilder<TemplateDataStoreVO> downloadTemplateSearch;
private SearchBuilder<TemplateDataStoreVO> uploadTemplateStateSearch;
+ private SearchBuilder<TemplateDataStoreVO> directDownloadTemplateSeach;
private SearchBuilder<VMTemplateVO> templateOnlySearch;
private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update template_store_ref set download_url_created=? where download_url_created is not null and store_id in (select id from image_store where data_center_id=?)";
@@ -143,6 +145,13 @@ public boolean configure(String name, Map<String, Object> params) throws Configu
downloadTemplateSearch.and("destroyed", downloadTemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
downloadTemplateSearch.done();
+ directDownloadTemplateSeach = createSearchBuilder();
+ directDownloadTemplateSeach.and("template_id", directDownloadTemplateSeach.entity().getTemplateId(), Op.EQ);
+ directDownloadTemplateSeach.and("download_state", directDownloadTemplateSeach.entity().getDownloadState(), Op.EQ);
+ directDownloadTemplateSeach.and("store_id", directDownloadTemplateSeach.entity().getDataStoreId(), Op.NULL);
+ directDownloadTemplateSeach.and("state", directDownloadTemplateSeach.entity().getState(), Op.EQ);
+ directDownloadTemplateSeach.done();
+
templateOnlySearch = _tmpltDao.createSearchBuilder();
templateOnlySearch.and("states", templateOnlySearch.entity().getState(), SearchCriteria.Op.IN);
uploadTemplateStateSearch = createSearchBuilder();
@@ -549,4 +558,34 @@ public void expireDnldUrlsForZone(Long dcId){
sc.setParameters("destroyed", false);
return listIncludingRemovedBy(sc);
}
+
+ @Override
+ public TemplateDataStoreVO createTemplateDirectDownloadEntry(long templateId, Long size) {
+ TemplateDataStoreVO templateDataStoreVO = new TemplateDataStoreVO();
+ templateDataStoreVO.setTemplateId(templateId);
+ templateDataStoreVO.setDataStoreRole(DataStoreRole.Image);
+ templateDataStoreVO.setState(State.Ready);
+ templateDataStoreVO.setDownloadState(Status.BYPASSED);
+ templateDataStoreVO.setSize(size == null ? 0l : size);
+ return templateDataStoreVO;
+ }
+
+ @Override
+ public TemplateDataStoreVO getReadyBypassedTemplate(long templateId) {
+ SearchCriteria<TemplateDataStoreVO> sc = directDownloadTemplateSeach.create();
+ sc.setParameters("template_id", templateId);
+ sc.setParameters("download_state", Status.BYPASSED);
+ sc.setParameters("state", State.Ready);
+ List<TemplateDataStoreVO> list = search(sc, null);
+ if (CollectionUtils.isEmpty(list) || list.size() > 1) {
+ return null;
+ }
+ return list.get(0);
+ }
+
+ @Override
+ public boolean isTemplateMarkedForDirectDownload(long templateId) {
+ TemplateDataStoreVO templateRef = getReadyBypassedTemplate(templateId);
+ return templateRef != null;
+ }
}
diff --git a/framework/direct-download/pom.xml b/framework/direct-download/pom.xml
new file mode 100644
index 00000000000..34a4aecdaba
--- /dev/null
+++ b/framework/direct-download/pom.xml
@@ -0,0 +1,31 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>cloud-framework-direct-download</artifactId>
+ <name>Apache CloudStack Framework - Direct Download to Primary Storage</name>
+ <parent>
+ <artifactId>cloudstack-framework</artifactId>
+ <groupId>org.apache.cloudstack</groupId>
+ <version>4.11.0.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+</project>
\ No newline at end of file
diff --git a/framework/direct-download/src/org/apache/cloudstack/framework/agent/direct/download/DirectDownloadService.java b/framework/direct-download/src/org/apache/cloudstack/framework/agent/direct/download/DirectDownloadService.java
new file mode 100644
index 00000000000..a9a96cc3483
--- /dev/null
+++ b/framework/direct-download/src/org/apache/cloudstack/framework/agent/direct/download/DirectDownloadService.java
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.cloudstack.framework.agent.direct.download;
+
+public interface DirectDownloadService {
+
+ /**
+ * Download template/ISO into poolId bypassing secondary storage. Download performed by hostId
+ */
+ void downloadTemplate(long templateId, long poolId, long hostId);
+
+ /**
+ * Upload client certificate to each running host
+ */
+ boolean uploadCertificateToHosts(String certificateCer, String certificateName);
+}
diff --git a/framework/pom.xml b/framework/pom.xml
index 5bfb1d094fb..66d60de2be1 100644
--- a/framework/pom.xml
+++ b/framework/pom.xml
@@ -56,5 +56,6 @@
<module>spring/lifecycle</module>
<module>spring/module</module>
<module>security</module>
+ <module>direct-download</module>
</modules>
</project>
diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec
index 898118fa96d..ef9d0278598 100644
--- a/packaging/centos63/cloud.spec
+++ b/packaging/centos63/cloud.spec
@@ -130,6 +130,8 @@ Requires: perl
Requires: libvirt-python
Requires: qemu-img
Requires: qemu-kvm
+Requires: epel-release
+Requires: aria2
Provides: cloud-agent
Obsoletes: cloud-agent < 4.1.0
Obsoletes: cloud-agent-libs < 4.1.0
diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec
index f16858a4a8f..78ee7d55199 100644
--- a/packaging/centos7/cloud.spec
+++ b/packaging/centos7/cloud.spec
@@ -111,6 +111,8 @@ Requires: perl
Requires: libvirt-python
Requires: qemu-img
Requires: qemu-kvm
+Requires: epel-release
+Requires: aria2
Provides: cloud-agent
Group: System Environment/Libraries
%description agent
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index 0ead0857492..398cf56e278 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -2200,8 +2200,18 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
KVMStoragePool pool = null;
final DataTO data = volume.getData();
if (volume.getType() == Volume.Type.ISO && data.getPath() != null) {
- final NfsTO nfsStore = (NfsTO)data.getDataStore();
- final String volPath = nfsStore.getUrl() + File.separator + data.getPath();
+ DataStoreTO dataStore = data.getDataStore();
+ String dataStoreUrl = null;
+ if (dataStore instanceof NfsTO) {
+ NfsTO nfsStore = (NfsTO)data.getDataStore();
+ dataStoreUrl = nfsStore.getUrl();
+ } else if (dataStore instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) dataStore).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+ //In order to support directly downloaded ISOs
+ String psHost = ((PrimaryDataStoreTO) dataStore).getHost();
+ String psPath = ((PrimaryDataStoreTO) dataStore).getPath();
+ dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
+ }
+ final String volPath = dataStoreUrl + File.separator + data.getPath();
final int index = volPath.lastIndexOf("/");
final String volDir = volPath.substring(0, index);
final String volName = volPath.substring(index + 1);
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
new file mode 100644
index 00000000000..efc009037b9
--- /dev/null
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
@@ -0,0 +1,50 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package com.cloud.hypervisor.kvm.resource.wrapper;
+
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
+import com.cloud.resource.CommandWrapper;
+import com.cloud.resource.ResourceWrapper;
+import com.cloud.utils.UriUtils;
+import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
+import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
+import org.apache.log4j.Logger;
+
+@ResourceWrapper(handles = CheckUrlCommand.class)
+public class LibvirtCheckUrlCommand extends CommandWrapper<CheckUrlCommand, CheckUrlAnswer, LibvirtComputingResource> {
+
+ private static final Logger s_logger = Logger.getLogger(LibvirtCheckUrlCommand.class);
+
+ @Override
+ public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serverResource) {
+ final String url = cmd.getUrl();
+ s_logger.info("Checking URL: " + url);
+ boolean checkResult = true;
+ Long remoteSize = null;
+ try {
+ UriUtils.checkUrlExistence(url);
+ remoteSize = UriUtils.getRemoteSize(url);
+ }
+ catch (IllegalArgumentException e) {
+ s_logger.warn(e.getMessage());
+ checkResult = false;
+ }
+ return new CheckUrlAnswer(checkResult, remoteSize);
+ }
+}
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
index 524fb35505a..72e0ca2ab2a 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
@@ -36,6 +36,18 @@
import javax.naming.ConfigurationException;
+import com.cloud.agent.direct.download.DirectTemplateDownloader;
+import com.cloud.agent.direct.download.DirectTemplateDownloader.DirectTemplateInformation;
+import com.cloud.agent.direct.download.HttpDirectTemplateDownloader;
+import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader;
+import com.cloud.agent.direct.download.NfsDirectTemplateDownloader;
+import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader;
+import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@@ -894,13 +906,21 @@ public Answer attachIso(final AttachCommand cmd) {
final DiskTO disk = cmd.getDisk();
final TemplateObjectTO isoTO = (TemplateObjectTO)disk.getData();
final DataStoreTO store = isoTO.getDataStore();
- if (!(store instanceof NfsTO)) {
+ String dataStoreUrl = null;
+ if (store instanceof NfsTO) {
+ NfsTO nfsStore = (NfsTO)store;
+ dataStoreUrl = nfsStore.getUrl();
+ } else if (store instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+ //In order to support directly downloaded ISOs
+ String psHost = ((PrimaryDataStoreTO) store).getHost();
+ String psPath = ((PrimaryDataStoreTO) store).getPath();
+ dataStoreUrl = "nfs://" + psHost + File.separator + psPath;
+ } else {
return new AttachAnswer("unsupported protocol");
}
- final NfsTO nfsStore = (NfsTO)store;
try {
final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
- attachOrDetachISO(conn, cmd.getVmName(), nfsStore.getUrl() + File.separator + isoTO.getPath(), true);
+ attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true);
} catch (final LibvirtException e) {
return new Answer(cmd, false, e.toString());
} catch (final URISyntaxException e) {
@@ -1387,4 +1407,37 @@ public Answer forgetObject(final ForgetObjectCmd cmd) {
return new Answer(cmd, false, "not implememented yet");
}
+ @Override
+ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+ final PrimaryDataStoreTO pool = cmd.getDestPool();
+ if (!pool.getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+ return new DirectDownloadAnswer(false, "Unsupported pool type " + pool.getPoolType().toString());
+ }
+ KVMStoragePool destPool = storagePoolMgr.getStoragePool(pool.getPoolType(), pool.getUuid());
+ DirectTemplateDownloader downloader;
+
+ if (cmd instanceof HttpDirectDownloadCommand) {
+ downloader = new HttpDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum(), ((HttpDirectDownloadCommand) cmd).getHeaders());
+ } else if (cmd instanceof HttpsDirectDownloadCommand) {
+ downloader = new HttpsDirectTemplateDownloader(cmd.getUrl(), cmd.getTemplateId(), destPool.getLocalPath(), cmd.getChecksum());
+ } else if (cmd instanceof NfsDirectDownloadCommand) {
+ downloader = new NfsDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
+ } else if (cmd instanceof MetalinkDirectDownloadCommand) {
+ downloader = new MetalinkDirectTemplateDownloader(cmd.getUrl(), destPool.getLocalPath(), cmd.getTemplateId(), cmd.getChecksum());
+ } else {
+ return new DirectDownloadAnswer(false, "Unsupported protocol, please provide HTTP(S), NFS or a metalink");
+ }
+
+ if (!downloader.downloadTemplate()) {
+ return new DirectDownloadAnswer(false, "Could not download template " + cmd.getTemplateId() + " on " + destPool.getLocalPath());
+ }
+ if (!downloader.validateChecksum()) {
+ return new DirectDownloadAnswer(false, "Checksum validation failed for template " + cmd.getTemplateId());
+ }
+ if (!downloader.extractAndInstallDownloadedTemplate()) {
+ return new DirectDownloadAnswer(false, "Template downloaded but there was an error on installation");
+ }
+ DirectTemplateInformation info = downloader.getTemplateInformation();
+ return new DirectDownloadAnswer(true, info.getSize(), info.getInstallPath());
+ }
}
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
index 96bb068c24f..6a0430f86ea 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
@@ -1341,4 +1341,5 @@ private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtEx
private void deleteDirVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException {
Script.runSimpleBashScript("rm -r --interactive=never " + vol.getPath());
}
+
}
diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
index dceb2910c57..43f94fbc57f 100644
--- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
+++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java
@@ -71,5 +71,4 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template,
public boolean deleteStoragePool(KVMStoragePool pool);
public boolean createFolder(String uuid, String path);
-
}
diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java
index 817c5f0a484..63d46bc87e9 100644
--- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java
+++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java
@@ -18,14 +18,34 @@
*/
package com.cloud.hypervisor.kvm.storage;
+import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource;
import javax.naming.ConfigurationException;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
public class KVMStorageProcessorTest {
+
+ @Mock
+ KVMStoragePoolManager storagePoolManager;
+ @Mock
+ LibvirtComputingResource resource;
+
+ private static final Long TEMPLATE_ID = 202l;
+ private static final String EXPECTED_DIRECT_DOWNLOAD_DIR = "template/2/202";
+
+ @Spy
+ @InjectMocks
+ private KVMStorageProcessor storageProcessor;
+
@Before
public void setUp() throws ConfigurationException {
+ MockitoAnnotations.initMocks(this);
+ storageProcessor = new KVMStorageProcessor(storagePoolManager, resource);
}
@Test
diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java
index 7c89921936a..5f43c38f607 100644
--- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java
+++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java
@@ -21,6 +21,7 @@
import java.net.URISyntaxException;
import java.util.UUID;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@@ -820,6 +821,11 @@ public ResignatureAnswer resignature(final ResignatureCommand cmd) {
return new ResignatureAnswer("Not implemented");
}
+ @Override
+ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+ return null;
+ }
+
/**
* Attach disks
* @param cmd
diff --git a/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java b/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java
old mode 100644
new mode 100755
index 9d86bc31b71..7ce222966f1
--- a/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java
+++ b/plugins/hypervisors/simulator/src/com/cloud/resource/SimulatorStorageProcessor.java
@@ -22,6 +22,7 @@
import java.io.File;
import java.util.UUID;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.log4j.Logger;
import org.apache.cloudstack.storage.command.AttachAnswer;
@@ -75,6 +76,11 @@ public ResignatureAnswer resignature(ResignatureCommand cmd) {
return new ResignatureAnswer();
}
+ @Override
+ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+ return null;
+ }
+
@Override
public Answer copyTemplateToPrimaryStorage(CopyCommand cmd) {
TemplateObjectTO template = new TemplateObjectTO();
diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java
index 012556eb1ca..ccf4512ce47 100644
--- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java
+++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java
@@ -34,6 +34,7 @@
import java.util.concurrent.TimeUnit;
import com.google.common.base.Strings;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@@ -2425,4 +2426,9 @@ public void setFullCloneFlag(boolean value){
this._fullCloneFlag = value;
s_logger.debug("VmwareProcessor instance - create full clone = " + (value ? "TRUE" : "FALSE"));
}
+
+ @Override
+ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+ return null;
+ }
}
diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
index d6697322857..0ff2a47f481 100644
--- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
+++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
@@ -44,6 +44,7 @@
import com.xensource.xenapi.VBD;
import com.xensource.xenapi.VDI;
import com.xensource.xenapi.VM;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@@ -195,6 +196,12 @@ public ResignatureAnswer resignature(final ResignatureCommand cmd) {
}
}
+ @Override
+ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
+ //Not implemented for Xen
+ return null;
+ }
+
@Override
public AttachAnswer attachIso(final AttachCommand cmd) {
final DiskTO disk = cmd.getDisk();
diff --git a/scripts/vm/hypervisor/kvm/setup_agent.sh b/scripts/vm/hypervisor/kvm/setup_agent.sh
index d55c6adfde2..b3c2e0fd280 100755
--- a/scripts/vm/hypervisor/kvm/setup_agent.sh
+++ b/scripts/vm/hypervisor/kvm/setup_agent.sh
@@ -224,5 +224,17 @@ then
setenforce 0
fi
+which aria2c
+if [ $? -gt 0 ]
+then
+ yum install epel-release -y
+ yum install aria2 -y
+ if [ $? -gt 0 ]
+ then
+ printf "failed to install aria2"
+ exit 1
+ fi
+fi
+
cloudstack-setup-agent --host=$host --zone=$zone --pod=$pod --cluster=$cluster --guid=$guid $paramters -a > /dev/null
#cloud_consoleP_setup $host $zone $pod
diff --git a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
index aacae264d1d..7d19b1e2f46 100644
--- a/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
+++ b/server/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
@@ -295,4 +295,5 @@
<bean id="annotationService" class="org.apache.cloudstack.annotation.AnnotationManagerImpl" />
+ <bean id="directDownloadManager" class="org.apache.cloudstack.direct.download.DirectDownloadManagerImpl" />
</beans>
diff --git a/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java
index bc9616a1bb5..26619f57e54 100644
--- a/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java
+++ b/server/src/com/cloud/api/query/dao/TemplateJoinDaoImpl.java
@@ -105,6 +105,8 @@ private String getTemplateStatus(TemplateJoinVO template) {
} else {
templateStatus = template.getDownloadPercent() + "% Downloaded";
}
+ } else if (template.getDownloadState() == Status.BYPASSED) {
+ templateStatus = "Bypassed Secondary Storage";
}else if (template.getErrorString()==null){
templateStatus = template.getTemplateState().toString();
}else {
@@ -197,6 +199,8 @@ public TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO te
addTagInformation(template, templateResponse);
}
+ templateResponse.setDirectDownload(template.isDirectDownload());
+
templateResponse.setObjectName("template");
return templateResponse;
}
@@ -320,6 +324,8 @@ public TemplateResponse newIsoResponse(TemplateJoinVO iso) {
} else {
isoStatus = iso.getDownloadPercent() + "% Downloaded";
}
+ } else if (iso.getDownloadState() == Status.BYPASSED) {
+ isoStatus = "Bypassed Secondary Storage";
} else {
isoStatus = iso.getErrorString();
}
@@ -348,6 +354,8 @@ public TemplateResponse newIsoResponse(TemplateJoinVO iso) {
}
}
+ isoResponse.setDirectDownload(iso.isDirectDownload());
+
isoResponse.setObjectName("iso");
return isoResponse;
diff --git a/server/src/com/cloud/api/query/vo/TemplateJoinVO.java b/server/src/com/cloud/api/query/vo/TemplateJoinVO.java
index 15a748bce41..20e805640ce 100644
--- a/server/src/com/cloud/api/query/vo/TemplateJoinVO.java
+++ b/server/src/com/cloud/api/query/vo/TemplateJoinVO.java
@@ -222,6 +222,9 @@
@Column(name = "temp_zone_pair")
private String tempZonePair; // represent a distinct (templateId, data_center_id) pair
+ @Column(name = "direct_download")
+ private boolean directDownload;
+
public TemplateJoinVO() {
}
@@ -477,4 +480,7 @@ public void setAccountId(long accountId) {
this.accountId = accountId;
}
+ public boolean isDirectDownload() {
+ return directDownload;
+ }
}
diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java
index c9916e90130..2966d41d8bf 100755
--- a/server/src/com/cloud/resource/ResourceManagerImpl.java
+++ b/server/src/com/cloud/resource/ResourceManagerImpl.java
@@ -25,6 +25,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
@@ -2797,6 +2798,23 @@ public void updateGPUDetails(final long hostId, final HashMap<String, HashMap<St
return null;
}
+ @Override
+ public HostVO findOneRandomRunningHostByHypervisor(HypervisorType type) {
+ final QueryBuilder<HostVO> sc = QueryBuilder.create(HostVO.class);
+ sc.and(sc.entity().getHypervisorType(), Op.EQ, type);
+ sc.and(sc.entity().getType(),Op.EQ, Type.Routing);
+ sc.and(sc.entity().getStatus(), Op.EQ, Status.Up);
+ sc.and(sc.entity().getResourceState(), Op.EQ, ResourceState.Enabled);
+ sc.and(sc.entity().getRemoved(), Op.NULL);
+ List<HostVO> hosts = sc.list();
+ if (CollectionUtils.isEmpty(hosts)) {
+ return null;
+ } else {
+ Collections.shuffle(hosts, new Random(System.currentTimeMillis()));
+ return hosts.get(0);
+ }
+ }
+
@Override
@DB
@ActionEvent(eventType = EventTypes.EVENT_HOST_RESERVATION_RELEASE, eventDescription = "releasing host reservation", async = true)
diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java
index 82a37529b25..6b33033cdd7 100644
--- a/server/src/com/cloud/server/ManagementServerImpl.java
+++ b/server/src/com/cloud/server/ManagementServerImpl.java
@@ -65,6 +65,7 @@
import org.apache.cloudstack.api.command.admin.config.ListHypervisorCapabilitiesCmd;
import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
import org.apache.cloudstack.api.command.admin.config.UpdateHypervisorCapabilitiesCmd;
+import org.apache.cloudstack.api.command.admin.direct.download.UploadTemplateDirectDownloadCertificate;
import org.apache.cloudstack.api.command.admin.domain.CreateDomainCmd;
import org.apache.cloudstack.api.command.admin.domain.DeleteDomainCmd;
import org.apache.cloudstack.api.command.admin.domain.ListDomainChildrenCmd;
@@ -3028,6 +3029,7 @@ public long getMemoryOrCpuCapacityByHost(final Long hostId, final short capacity
cmdList.add(ReleasePodIpCmdByAdmin.class);
cmdList.add(CreateManagementNetworkIpRangeCmd.class);
cmdList.add(DeleteManagementNetworkIpRangeCmd.class);
+ cmdList.add(UploadTemplateDirectDownloadCertificate.class);
// Out-of-band management APIs for admins
cmdList.add(EnableOutOfBandManagementForHostCmd.class);
@@ -3040,7 +3042,6 @@ public long getMemoryOrCpuCapacityByHost(final Long hostId, final short capacity
cmdList.add(IssueOutOfBandManagementPowerActionCmd.class);
cmdList.add(ChangeOutOfBandManagementPasswordCmd.class);
cmdList.add(GetUserKeysCmd.class);
-
return cmdList;
}
diff --git a/server/src/com/cloud/storage/TemplateProfile.java b/server/src/com/cloud/storage/TemplateProfile.java
index 1ca1d81fe8f..410b519968a 100644
--- a/server/src/com/cloud/storage/TemplateProfile.java
+++ b/server/src/com/cloud/storage/TemplateProfile.java
@@ -51,6 +51,8 @@
Map details;
Boolean isDynamicallyScalable;
TemplateType templateType;
+ Boolean directDownload;
+ Long size;
public TemplateProfile(Long templateId, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List<Long> zoneIdList, HypervisorType hypervisorType,
@@ -93,7 +95,7 @@ public TemplateProfile(Long templateId, Long userId, String name, String display
Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List<Long> zoneId,
HypervisorType hypervisorType, String accountName, Long domainId, Long accountId, String chksum, Boolean bootable, String templateTag, Map details,
- Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType) {
+ Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload) {
this(templateId,
userId,
name,
@@ -119,6 +121,7 @@ public TemplateProfile(Long templateId, Long userId, String name, String display
this.templateTag = templateTag;
this.isDynamicallyScalable = isDynamicallyScalable;
this.templateType = templateType;
+ this.directDownload = directDownload;
}
public Long getTemplateId() {
@@ -316,4 +319,16 @@ public TemplateType getTemplateType() {
public void setTemplateType(TemplateType templateType) {
this.templateType = templateType;
}
+
+ public boolean isDirectDownload() {
+ return directDownload == null ? false : directDownload;
+ }
+
+ public Long getSize() {
+ return size;
+ }
+
+ public void setSize(Long size) {
+ this.size = size;
+ }
}
diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java
index 49039d14b90..fedc0a6d83b 100644
--- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java
+++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java
@@ -16,6 +16,10 @@
// under the License.
package com.cloud.template;
+import com.cloud.agent.api.Answer;
+import com.cloud.host.HostVO;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.resource.ResourceManager;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
@@ -29,6 +33,8 @@
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionStatus;
+import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer;
+import org.apache.cloudstack.agent.directdownload.CheckUrlCommand;
import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@@ -68,6 +74,9 @@
import com.cloud.exception.ResourceAllocationException;
import com.cloud.org.Grouping;
import com.cloud.server.StatsCollector;
+import com.cloud.template.VirtualMachineTemplate.State;
+import com.cloud.user.Account;
+import com.cloud.utils.Pair;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
@@ -77,9 +86,6 @@
import com.cloud.storage.VMTemplateZoneVO;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.download.DownloadMonitor;
-import com.cloud.template.VirtualMachineTemplate.State;
-import com.cloud.user.Account;
-import com.cloud.utils.Pair;
import com.cloud.utils.UriUtils;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.EntityManager;
@@ -113,17 +119,42 @@
DataCenterDao _dcDao;
@Inject
MessageBus _messageBus;
+ @Inject
+ ResourceManager resourceManager;
@Override
public String getName() {
return TemplateAdapterType.Hypervisor.getName();
}
+ /**
+ * Validate on random running KVM host that URL is reachable
+ * @param url url
+ */
+ private Long performDirectDownloadUrlValidation(final String url) {
+ HostVO host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM);
+ if (host == null) {
+ throw new CloudRuntimeException("Couldn't find a host to validate URL " + url);
+ }
+ CheckUrlCommand cmd = new CheckUrlCommand(url);
+ s_logger.debug("Performing URL " + url + " validation on host " + host.getId());
+ Answer answer = _agentMgr.easySend(host.getId(), cmd);
+ if (answer == null || !answer.getResult()) {
+ throw new CloudRuntimeException("URL: " + url + " validation failed on host id " + host.getId());
+ }
+ CheckUrlAnswer ans = (CheckUrlAnswer) answer;
+ return ans.getTemplateSize();
+ }
+
@Override
public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationException {
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(ImageFormat.ISO.getFileExtension(), url);
+ if (cmd.isDirectDownload()) {
+ Long templateSize = performDirectDownloadUrlValidation(url);
+ profile.setSize(templateSize);
+ }
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
@@ -135,6 +166,10 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio
TemplateProfile profile = super.prepare(cmd);
String url = profile.getUrl();
UriUtils.validateUrl(cmd.getFormat(), url);
+ if (cmd.isDirectDownload()) {
+ Long templateSize = performDirectDownloadUrlValidation(url);
+ profile.setSize(templateSize);
+ }
profile.setUrl(url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(cmd.getEntityOwnerId()), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
@@ -150,6 +185,14 @@ public TemplateProfile prepare(GetUploadParamsForTemplateCmd cmd) throws Resourc
return profile;
}
+ /**
+ * Persist template marking it for direct download to Primary Storage, skipping Secondary Storage
+ */
+ private void persistDirectDownloadTemplate(long templateId, Long size) {
+ TemplateDataStoreVO directDownloadEntry = templateDataStoreDao.createTemplateDirectDownloadEntry(templateId, size);
+ templateDataStoreDao.persist(directDownloadEntry);
+ }
+
@Override
public VMTemplateVO create(TemplateProfile profile) {
// persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync.
@@ -159,17 +202,23 @@ public VMTemplateVO create(TemplateProfile profile) {
throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate());
}
- List<Long> zones = profile.getZoneIdList();
+ if (!profile.isDirectDownload()) {
+ List<Long> zones = profile.getZoneIdList();
- //zones is null when this template is to be registered to all zones
- if (zones == null){
- createTemplateWithinZone(null, profile, template);
- }
- else {
- for (Long zId : zones) {
- createTemplateWithinZone(zId, profile, template);
+ //zones is null when this template is to be registered to all zones
+ if (zones == null){
+ createTemplateWithinZone(null, profile, template);
+ }
+ else {
+ for (Long zId : zones) {
+ createTemplateWithinZone(zId, profile, template);
+ }
}
+ } else {
+ //KVM direct download templates bypassing Secondary Storage
+ persistDirectDownloadTemplate(template.getId(), profile.getSize());
}
+
_resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template);
return template;
}
diff --git a/server/src/com/cloud/template/TemplateAdapter.java b/server/src/com/cloud/template/TemplateAdapter.java
index 7233ac98751..595de66ed17 100644
--- a/server/src/com/cloud/template/TemplateAdapter.java
+++ b/server/src/com/cloud/template/TemplateAdapter.java
@@ -71,11 +71,11 @@ public String getName() {
public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String accountName,
- Long domainId, String chksum, Boolean bootable, Map details) throws ResourceAllocationException;
+ Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException;
public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String chksum,
Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable,
- TemplateType templateType) throws ResourceAllocationException;
+ TemplateType templateType, boolean directDownload) throws ResourceAllocationException;
}
diff --git a/server/src/com/cloud/template/TemplateAdapterBase.java b/server/src/com/cloud/template/TemplateAdapterBase.java
index 1e2c6d2178a..7467e30aa01 100644
--- a/server/src/com/cloud/template/TemplateAdapterBase.java
+++ b/server/src/com/cloud/template/TemplateAdapterBase.java
@@ -112,6 +112,8 @@
ConfigurationServer _configServer;
@Inject
ProjectManager _projectMgr;
+ @Inject
+ private TemplateDataStoreDao templateDataStoreDao;
@Override
public boolean stop() {
@@ -121,16 +123,16 @@ public boolean stop() {
@Override
public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneId, HypervisorType hypervisorType, String accountName,
- Long domainId, String chksum, Boolean bootable, Map details) throws ResourceAllocationException {
+ Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException {
return prepare(isIso, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, format, guestOSId, zoneId,
- hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER);
+ hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload);
}
@Override
public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url,
Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List<Long> zoneIdList, HypervisorType hypervisorType, String chksum,
Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshkeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable,
- TemplateType templateType) throws ResourceAllocationException {
+ TemplateType templateType, boolean directDownload) throws ResourceAllocationException {
//Long accountId = null;
// parameters verification
@@ -249,7 +251,7 @@ public TemplateProfile prepare(boolean isIso, long userId, String name, String d
CallContext.current().setEventDetails("Id: " + id + " name: " + name);
return new TemplateProfile(id, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList,
hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details,
- sshkeyEnabled, null, isDynamicallyScalable, templateType);
+ sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload);
}
@@ -277,7 +279,7 @@ public TemplateProfile prepare(RegisterTemplateCmd cmd) throws ResourceAllocatio
return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(),
cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true,
- cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER);
+ cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload());
}
@@ -308,7 +310,7 @@ public TemplateProfile prepare(GetUploadParamsForTemplateCmd cmd) throws Resourc
return prepare(false, CallContext.current().getCallingUserId(), cmd.getName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(),
cmd.getRequiresHvm(), null, cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneList,
hypervisorType, cmd.getChecksum(), true, cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null,
- cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER);
+ cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, false);
}
@@ -330,7 +332,7 @@ public TemplateProfile prepare(RegisterIsoCmd cmd) throws ResourceAllocationExce
return prepare(true, CallContext.current().getCallingUserId(), cmd.getIsoName(), cmd.getDisplayText(), 64, false, true, cmd.getUrl(), cmd.isPublic(),
cmd.isFeatured(), cmd.isExtractable(), ImageFormat.ISO.toString(), cmd.getOsTypeId(), zoneList, HypervisorType.None, cmd.getChecksum(), cmd.isBootable(), null,
- owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER);
+ owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload());
}
protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTemplate.State initialState) {
@@ -339,9 +341,13 @@ protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTe
new VMTemplateVO(profile.getTemplateId(), profile.getName(), profile.getFormat(), profile.getIsPublic(), profile.getFeatured(), profile.getIsExtractable(),
profile.getTemplateType(), profile.getUrl(), profile.getRequiresHVM(), profile.getBits(), profile.getAccountId(), profile.getCheckSum(),
profile.getDisplayText(), profile.getPasswordEnabled(), profile.getGuestOsId(), profile.getBootable(), profile.getHypervisorType(),
- profile.getTemplateTag(), profile.getDetails(), profile.getSshKeyEnabled(), profile.IsDynamicallyScalable());
+ profile.getTemplateTag(), profile.getDetails(), profile.getSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload());
template.setState(initialState);
+ if (profile.isDirectDownload()) {
+ template.setSize(profile.getSize());
+ }
+
if (zoneIdList == null) {
List<DataCenterVO> dcs = _dcDao.listAll();
diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java
index 41f11af68d0..270194ff0b8 100755
--- a/server/src/com/cloud/template/TemplateManagerImpl.java
+++ b/server/src/com/cloud/template/TemplateManagerImpl.java
@@ -32,6 +32,7 @@
import javax.inject.Inject;
import javax.naming.ConfigurationException;
+import com.cloud.deploy.DeployDestination;
import com.cloud.storage.ImageStoreUploadMonitorImpl;
import com.cloud.utils.StringUtils;
import com.cloud.utils.EncryptionUtil;
@@ -551,10 +552,18 @@ private String extract(Account caller, Long templateId, String url, Long zoneId,
}
@Override
- public void prepareIsoForVmProfile(VirtualMachineProfile profile) {
+ public void prepareIsoForVmProfile(VirtualMachineProfile profile, DeployDestination dest) {
UserVmVO vm = _userVmDao.findById(profile.getId());
if (vm.getIsoId() != null) {
- TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId());
+ Map<Volume, StoragePool> storageForDisks = dest.getStorageForDisks();
+ Long poolId = null;
+ for (StoragePool storagePool : storageForDisks.values()) {
+ if (poolId != null && storagePool.getId() != poolId) {
+ throw new CloudRuntimeException("Cannot determine where to download iso");
+ }
+ poolId = storagePool.getId();
+ }
+ TemplateInfo template = prepareIso(vm.getIsoId(), vm.getDataCenterId(), dest.getHost().getId(), poolId);
if (template == null){
s_logger.error("Failed to prepare ISO on secondary or cache storage");
throw new CloudRuntimeException("Failed to prepare ISO on secondary or cache storage");
@@ -1156,14 +1165,22 @@ public boolean attachIso(long isoId, long vmId) {
// for ISO, we need to consider whether to copy to cache storage or not if it is not on NFS, since our hypervisor resource always assumes that they are in NFS
@Override
- public TemplateInfo prepareIso(long isoId, long dcId) {
- TemplateInfo tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId);
+ public TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId) {
+ TemplateInfo tmplt;
+ boolean bypassed = false;
+ if (_tmplFactory.isTemplateMarkedForDirectDownload(isoId)) {
+ tmplt = _tmplFactory.getReadyBypassedTemplateOnPrimaryStore(isoId, poolId, hostId);
+ bypassed = true;
+ } else {
+ tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId);
+ }
+
if (tmplt == null || tmplt.getFormat() != ImageFormat.ISO) {
s_logger.warn("ISO: " + isoId + " does not exist in vm_template table");
return null;
}
- if (tmplt.getDataStore() != null && !(tmplt.getDataStore().getTO() instanceof NfsTO)) {
+ if (!bypassed && tmplt.getDataStore() != null && !(tmplt.getDataStore().getTO() instanceof NfsTO)) {
// if it is s3, need to download into cache storage first
Scope destScope = new ZoneScope(dcId);
TemplateInfo cacheData = (TemplateInfo)cacheMgr.createCacheObject(tmplt, destScope);
@@ -1187,7 +1204,7 @@ private boolean attachISOToVM(long vmId, long isoId, boolean attach) {
}
// prepare ISO ready to mount on hypervisor resource level
- TemplateInfo tmplt = prepareIso(isoId, vm.getDataCenterId());
+ TemplateInfo tmplt = prepareIso(isoId, vm.getDataCenterId(), vm.getHostId(), null);
String vmName = vm.getInstanceName();
@@ -1804,7 +1821,7 @@ public VMTemplateVO createPrivateTemplateRecord(CreateTemplateCmd cmd, Account t
}
privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable,
TemplateType.USER, null, requiresHvmValue, bitsValue, templateOwner.getId(), null, description,
- passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), false, isDynamicScalingEnabled);
+ passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), false, isDynamicScalingEnabled, false);
if (sourceTemplateId != null) {
if (s_logger.isDebugEnabled()) {
@@ -1912,6 +1929,10 @@ public DataStore getImageStore(long tmpltId) {
@Override
public Long getTemplateSize(long templateId, long zoneId) {
+ if (_tmplStoreDao.isTemplateMarkedForDirectDownload(templateId)) {
+ // check if template is marked for direct download
+ return _tmplStoreDao.getReadyBypassedTemplate(templateId).getSize();
+ }
TemplateDataStoreVO templateStoreRef = _tmplStoreDao.findByTemplateZoneDownloadStatus(templateId, zoneId, VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
if (templateStoreRef == null) {
// check if it is ready on image cache stores
diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java
index df50f5a9162..3f351d6de40 100644
--- a/server/src/com/cloud/vm/UserVmManagerImpl.java
+++ b/server/src/com/cloud/vm/UserVmManagerImpl.java
@@ -4047,7 +4047,7 @@ public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, Depl
- _templateMgr.prepareIsoForVmProfile(profile);
+ _templateMgr.prepareIsoForVmProfile(profile, dest);
return true;
}
diff --git a/server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
new file mode 100755
index 00000000000..6aa7ad1b914
--- /dev/null
+++ b/server/src/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java
@@ -0,0 +1,245 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.api.Answer;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status;
+import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import javax.inject.Inject;
+
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
+import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand;
+import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
+import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+import org.apache.commons.collections.MapUtils;
+import org.apache.log4j.Logger;
+
+public class DirectDownloadManagerImpl extends ManagerBase implements DirectDownloadManager {
+
+ private static final Logger s_logger = Logger.getLogger(DirectDownloadManagerImpl.class);
+ protected static final String httpHeaderDetailKey = "HTTP_HEADER";
+ protected static final String BEGIN_CERT = "-----BEGIN CERTIFICATE-----";
+ protected static final String END_CERT = "-----END CERTIFICATE-----";
+ protected final static String LINE_SEPARATOR = "\n";
+
+ @Inject
+ VMTemplateDao vmTemplateDao;
+ @Inject
+ PrimaryDataStoreDao primaryDataStoreDao;
+ @Inject
+ HostDao hostDao;
+ @Inject
+ AgentManager agentManager;
+ @Inject
+ VMTemplatePoolDao vmTemplatePoolDao;
+ @Inject
+ DataStoreManager dataStoreManager;
+
+ @Override
+ public List<Class<?>> getCommands() {
+ final List<Class<?>> cmdList = new ArrayList<Class<?>>();
+ return cmdList;
+ }
+
+ /**
+ * Return protocol to use from provided URL
+ * @param url
+ * @return
+ */
+ public static DownloadProtocol getProtocolFromUrl(String url) {
+ URI uri;
+ try {
+ uri = new URI(url);
+ } catch (URISyntaxException e) {
+ throw new CloudRuntimeException("URI is incorrect: " + url);
+ }
+ if ((uri != null) && (uri.getScheme() != null)) {
+ if (uri.getPath().endsWith(".metalink")) {
+ return DownloadProtocol.METALINK;
+ } else if (uri.getScheme().equalsIgnoreCase("http")) {
+ return DownloadProtocol.HTTP;
+ } else if (uri.getScheme().equalsIgnoreCase("https")) {
+ return DownloadProtocol.HTTPS;
+ } else if (uri.getScheme().equalsIgnoreCase("nfs")) {
+ return DownloadProtocol.NFS;
+ } else {
+ throw new CloudRuntimeException("Scheme is not supported " + url);
+ }
+ } else {
+ throw new CloudRuntimeException("URI is incorrect: " + url);
+ }
+ }
+
+ /**
+ * Return HTTP headers from template details
+ * @param templateDetails
+ * @return
+ */
+ protected Map<String, String> getHeadersFromDetails(Map<String, String> templateDetails) {
+ if (MapUtils.isEmpty(templateDetails)) {
+ return new HashMap<>();
+ }
+ Map<String, String> headers = new HashMap<>();
+ for (String key : templateDetails.keySet()) {
+ if (key.startsWith(httpHeaderDetailKey)) {
+ String header = key.split(":")[1];
+ String value = templateDetails.get(key);
+ headers.put(header, value);
+ }
+ }
+ return headers;
+ }
+
+ @Override
+ public void downloadTemplate(long templateId, long poolId, long hostId) {
+ VMTemplateVO template = vmTemplateDao.findById(templateId);
+ StoragePoolVO pool = primaryDataStoreDao.findById(poolId);
+ HostVO host = hostDao.findById(hostId);
+ if (pool == null) {
+ throw new CloudRuntimeException("Storage pool " + poolId + " could not be found");
+ }
+ if (template == null) {
+ throw new CloudRuntimeException("Template " + templateId + " could not be found");
+ }
+ if (host == null) {
+ throw new CloudRuntimeException("Host " + hostId + " could not be found");
+ }
+ if (!template.isDirectDownload()) {
+ throw new CloudRuntimeException("Template " + templateId + " is not marked for direct download");
+ }
+ Map<String, String> details = template.getDetails();
+ String url = template.getUrl();
+ String checksum = template.getChecksum();
+ Map<String, String> headers = getHeadersFromDetails(details);
+ DataStore store = dataStoreManager.getDataStore(poolId, DataStoreRole.Primary);
+ if (store == null) {
+ throw new CloudRuntimeException("Data store " + poolId + " could not be found");
+ }
+ PrimaryDataStore primaryDataStore = (PrimaryDataStore) store;
+ PrimaryDataStoreTO to = (PrimaryDataStoreTO) primaryDataStore.getTO();
+
+ DownloadProtocol protocol = getProtocolFromUrl(url);
+ DirectDownloadCommand cmd = getDirectDownloadCommandFromProtocol(protocol, url, templateId, to, checksum, headers);
+ Answer answer = agentManager.easySend(hostId, cmd);
+ if (answer == null || !answer.getResult()) {
+ throw new CloudRuntimeException("Host " + hostId + " could not download template " +
+ templateId + " on pool " + poolId);
+ }
+
+ VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId);
+ if (sPoolRef == null) {
+ if (s_logger.isDebugEnabled()) {
+ s_logger.debug("Not found (templateId:" + templateId + " poolId: " + poolId + ") in template_spool_ref, persisting it");
+ }
+ DirectDownloadAnswer ans = (DirectDownloadAnswer) answer;
+ sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId);
+ sPoolRef.setDownloadPercent(100);
+ sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
+ sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready);
+ sPoolRef.setTemplateSize(ans.getTemplateSize());
+ sPoolRef.setLocalDownloadPath(ans.getInstallPath());
+ sPoolRef.setInstallPath(ans.getInstallPath());
+ vmTemplatePoolDao.persist(sPoolRef);
+ }
+ }
+
+ /**
+ * Return DirectDownloadCommand according to the protocol
+ * @param protocol
+ * @param url
+ * @param templateId
+ * @param destPool
+ * @return
+ */
+ private DirectDownloadCommand getDirectDownloadCommandFromProtocol(DownloadProtocol protocol, String url, Long templateId, PrimaryDataStoreTO destPool,
+ String checksum, Map<String, String> httpHeaders) {
+ if (protocol.equals(DownloadProtocol.HTTP)) {
+ return new HttpDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
+ } else if (protocol.equals(DownloadProtocol.HTTPS)) {
+ return new HttpsDirectDownloadCommand(url, templateId, destPool, checksum, httpHeaders);
+ } else if (protocol.equals(DownloadProtocol.NFS)) {
+ return new NfsDirectDownloadCommand(url, templateId, destPool, checksum);
+ } else if (protocol.equals(DownloadProtocol.METALINK)) {
+ return new MetalinkDirectDownloadCommand(url, templateId, destPool, checksum);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public boolean uploadCertificateToHosts(String certificateCer, String certificateName) {
+ List<HostVO> hosts = hostDao.listAllHostsByType(Host.Type.Routing)
+ .stream()
+ .filter(x -> x.getStatus().equals(Status.Up) &&
+ x.getHypervisorType().equals(Hypervisor.HypervisorType.KVM))
+ .collect(Collectors.toList());
+ for (HostVO host : hosts) {
+ if (!uploadCertificate(certificateCer, certificateName, host.getId())) {
+ throw new CloudRuntimeException("Uploading certificate " + certificateName + " failed on host: " + host.getId());
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Upload and import certificate to hostId on keystore
+ */
+ protected boolean uploadCertificate(String certificate, String certificateName, long hostId) {
+ String cert = certificate.replaceAll("(.{64})", "$1\n");
+ final String prettified_cert = BEGIN_CERT + LINE_SEPARATOR + cert + LINE_SEPARATOR + END_CERT;
+ SetupDirectDownloadCertificate cmd = new SetupDirectDownloadCertificate(prettified_cert, certificateName);
+ Answer answer = agentManager.easySend(hostId, cmd);
+ if (answer == null || !answer.getResult()) {
+ return false;
+ }
+ s_logger.info("Certificate " + certificateName + " successfully uploaded to host: " + hostId);
+ return true;
+ }
+}
diff --git a/server/test/com/cloud/resource/MockResourceManagerImpl.java b/server/test/com/cloud/resource/MockResourceManagerImpl.java
index 3eb5f519e64..e3e46de7ac1 100755
--- a/server/test/com/cloud/resource/MockResourceManagerImpl.java
+++ b/server/test/com/cloud/resource/MockResourceManagerImpl.java
@@ -614,6 +614,12 @@ public void updateGPUDetails(final long hostId, final HashMap<String, HashMap<St
return null;
}
+ @Override
+ public HostVO findOneRandomRunningHostByHypervisor(HypervisorType type) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
@Override
public boolean isHostGpuEnabled(final long hostId) {
// TODO Auto-generated method stub
diff --git a/server/test/org/apache/cloudstack/direct/download/DirectDownloadManagerImplTest.java b/server/test/org/apache/cloudstack/direct/download/DirectDownloadManagerImplTest.java
new file mode 100644
index 00000000000..59405998f9f
--- /dev/null
+++ b/server/test/org/apache/cloudstack/direct/download/DirectDownloadManagerImplTest.java
@@ -0,0 +1,106 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.cloudstack.direct.download;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.host.dao.HostDao;
+import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand.DownloadProtocol;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DirectDownloadManagerImplTest {
+
+ @Mock
+ HostDao hostDao;
+ @Mock
+ AgentManager agentManager;
+
+ @Spy
+ @InjectMocks
+ private DirectDownloadManagerImpl manager = new DirectDownloadManagerImpl();
+
+ private static final String HTTP_HEADER_1 = "Content-Type";
+ private static final String HTTP_VALUE_1 = "application/x-www-form-urlencoded";
+ private static final String HTTP_HEADER_2 = "Accept-Encoding";
+ private static final String HTTP_VALUE_2 = "gzip";
+
+ @Before
+ public void setUp() {
+ }
+
+ @Test
+ public void testGetProtocolMetalink() {
+ String url = "http://192.168.1.2/tmpl.metalink";
+ DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+ Assert.assertEquals(DownloadProtocol.METALINK, protocol);
+ }
+
+ @Test
+ public void testGetProtocolHttp() {
+ String url = "http://192.168.1.2/tmpl.qcow2";
+ DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+ Assert.assertEquals(DownloadProtocol.HTTP, protocol);
+ }
+
+ @Test
+ public void testGetProtocolHttps() {
+ String url = "https://192.168.1.2/tmpl.qcow2";
+ DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+ Assert.assertEquals(DownloadProtocol.HTTPS, protocol);
+ }
+
+ @Test
+ public void testGetProtocolNfs() {
+ String url = "nfs://192.168.1.2/tmpl.qcow2";
+ DownloadProtocol protocol = DirectDownloadManagerImpl.getProtocolFromUrl(url);
+ Assert.assertEquals(DownloadProtocol.NFS, protocol);
+ }
+
+ @Test
+ public void testGetHeadersFromDetailsHttpHeaders() {
+ Map<String, String> details = new HashMap<>();
+ details.put("Message.ReservedCapacityFreed.Flag", "false");
+ details.put(DirectDownloadManagerImpl.httpHeaderDetailKey + ":" + HTTP_HEADER_1, HTTP_VALUE_1);
+ details.put(DirectDownloadManagerImpl.httpHeaderDetailKey + ":" + HTTP_HEADER_2, HTTP_VALUE_2);
+ Map<String, String> headers = manager.getHeadersFromDetails(details);
+ Assert.assertEquals(2, headers.keySet().size());
+ Assert.assertTrue(headers.containsKey(HTTP_HEADER_1));
+ Assert.assertTrue(headers.containsKey(HTTP_HEADER_2));
+ Assert.assertEquals(HTTP_VALUE_1, headers.get(HTTP_HEADER_1));
+ Assert.assertEquals(HTTP_VALUE_2, headers.get(HTTP_HEADER_2));
+ }
+
+ @Test
+ public void testGetHeadersFromDetailsNonHttpHeaders() {
+ Map<String, String> details = new HashMap<>();
+ details.put("Message.ReservedCapacityFreed.Flag", "false");
+ Map<String, String> headers = manager.getHeadersFromDetails(details);
+ Assert.assertTrue(headers.isEmpty());
+ }
+}
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
index dd9df21adf8..5fa13a34cfc 100644
--- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -38,6 +38,23 @@
import javax.naming.ConfigurationException;
+import com.cloud.storage.template.Processor;
+import com.cloud.storage.template.S3TemplateDownloader;
+import com.cloud.storage.template.TemplateDownloader;
+import com.cloud.storage.template.TemplateLocation;
+import com.cloud.storage.template.MetalinkTemplateDownloader;
+import com.cloud.storage.template.HttpTemplateDownloader;
+import com.cloud.storage.template.LocalTemplateDownloader;
+import com.cloud.storage.template.ScpTemplateDownloader;
+import com.cloud.storage.template.TemplateProp;
+import com.cloud.storage.template.OVAProcessor;
+import com.cloud.storage.template.IsoProcessor;
+import com.cloud.storage.template.QCOW2Processor;
+import com.cloud.storage.template.VmdkProcessor;
+import com.cloud.storage.template.RawImageProcessor;
+import com.cloud.storage.template.TARProcessor;
+import com.cloud.storage.template.VhdProcessor;
+import com.cloud.storage.template.TemplateConstants;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
@@ -56,25 +73,9 @@
import com.cloud.storage.StorageLayer;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
-import com.cloud.storage.template.HttpTemplateDownloader;
-import com.cloud.storage.template.IsoProcessor;
-import com.cloud.storage.template.LocalTemplateDownloader;
-import com.cloud.storage.template.OVAProcessor;
-import com.cloud.storage.template.Processor;
import com.cloud.storage.template.Processor.FormatInfo;
-import com.cloud.storage.template.QCOW2Processor;
-import com.cloud.storage.template.RawImageProcessor;
-import com.cloud.storage.template.S3TemplateDownloader;
-import com.cloud.storage.template.ScpTemplateDownloader;
-import com.cloud.storage.template.TARProcessor;
-import com.cloud.storage.template.TemplateConstants;
-import com.cloud.storage.template.TemplateDownloader;
import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
import com.cloud.storage.template.TemplateDownloader.Status;
-import com.cloud.storage.template.TemplateLocation;
-import com.cloud.storage.template.TemplateProp;
-import com.cloud.storage.template.VhdProcessor;
-import com.cloud.storage.template.VmdkProcessor;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.StringUtils;
import com.cloud.utils.component.ManagerBase;
@@ -560,7 +561,9 @@ public String downloadPublicTemplate(long id, String url, String name, ImageForm
}
TemplateDownloader td;
if ((uri != null) && (uri.getScheme() != null)) {
- if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
+ if (uri.getPath().endsWith(".metalink")) {
+ td = new MetalinkTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes);
+ } else if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
} else if (uri.getScheme().equalsIgnoreCase("file")) {
td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
diff --git a/test/integration/smoke/test_templates.py b/test/integration/smoke/test_templates.py
index 43d7233fd68..ba7a3335b5f 100644
--- a/test/integration/smoke/test_templates.py
+++ b/test/integration/smoke/test_templates.py
@@ -1197,3 +1197,163 @@ def test_09_copy_delete_template(self):
"Removed state is not correct."
)
return
+
+class TestCreateTemplateWithDirectDownload(cloudstackTestCase):
+
+ @classmethod
+ def setUpClass(self):
+ self.testClient = super(TestCreateTemplateWithDirectDownload, self).getClsTestClient()
+ self.apiclient = self.testClient.getApiClient()
+ self.dbclient = self.testClient.getDbConnection()
+ self._cleanup = []
+ self.templates = []
+
+ self.services = self.testClient.getParsedTestDataConfig()
+ self.unsupportedHypervisor = False
+ self.hypervisor = self.testClient.getHypervisorInfo()
+ if self.hypervisor.lower() not in ['kvm']:
+ # Direct Download is only available for KVM hypervisor
+ self.unsupportedHypervisor = True
+ self.skipTest("Skipping test because unsupported hypervisor\
+ %s" % self.hypervisor)
+ return
+
+ # Get Zone, Domain and templates
+ self.domain = get_domain(self.apiclient)
+ self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests())
+ self.services["mode"] = self.zone.networktype
+ self.services["virtual_machine"]["zoneid"] = self.zone.id
+ self.account = Account.create(
+ self.apiclient,
+ self.services["account"],
+ admin=True,
+ domainid=self.domain.id
+ )
+ self._cleanup.append(self.account)
+ self.user = Account.create(
+ self.apiclient,
+ self.services["account"],
+ domainid=self.domain.id
+ )
+ self._cleanup.append(self.user)
+ self.service_offering = ServiceOffering.create(
+ self.apiclient,
+ self.services["service_offerings"]["tiny"]
+ )
+ self._cleanup.append(self.service_offering)
+
+ self.template = {
+ "name": "tiny-kvm",
+ "displaytext": "tiny kvm",
+ "format": "QCOW2",
+ "url": "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2",
+ "requireshvm": "True",
+ "ispublic": "True",
+ "isextractable": "True",
+ "checksum": "{SHA-1}" + "6952e58f39b470bd166ace11ffd20bf479bed936",
+ "hypervisor": self.hypervisor,
+ "zoneid": self.zone.id,
+ "ostype": "Other Linux (64-bit)",
+ "directdownload": True
+ }
+
+ return
+
+ @classmethod
+ def tearDownClass(cls):
+ try:
+ cleanup_resources(cls.apiclient, cls._cleanup)
+ except Exception as e:
+ raise Exception("Warning: Exception during cleanup : %s" % e)
+ return
+
+ def setUp(self):
+ self.apiclient = self.testClient.getApiClient()
+ self.dbclient = self.testClient.getDbConnection()
+ self.cleanup = []
+
+ if self.unsupportedHypervisor:
+ self.skipTest("Skipping test because unsupported hypervisor\
+ %s" % self.hypervisor)
+ return
+
+ def tearDown(self):
+ try:
+ #Clean up, terminate the created templates
+ cleanup_resources(self.apiclient, self.cleanup)
+
+ except Exception as e:
+ raise Exception("Warning: Exception during cleanup : %s" % e)
+ return
+
+ @attr(tags=["advanced", "smoke"], required_hardware="true")
+ def test_01_register_template_direct_download_flag(self):
+ """
+ Register a template using Direct Download flag
+ """
+ self.bypassed_template = Template.register(self.apiclient, self.template, zoneid=self.zone.id, hypervisor=self.hypervisor, randomize_name=False)
+ self._cleanup.append(self.bypassed_template)
+ self.templates.append(self.bypassed_template)
+
+ tmplt = self.dbclient.execute("select id, direct_download from vm_template where uuid='%s';" % self.bypassed_template.id)
+ det = tmplt[0]
+
+ self.assertEqual(det[1],
+ 1,
+ "Template should be marked as Direct Download"
+ )
+ qresultset = self.dbclient.execute("select download_state, state from template_store_ref where template_id='%s' and store_id is NULL;"
+ % det[0])
+ ref = qresultset[0]
+ self.assertEqual(ref[0],
+ "BYPASSED",
+ "Template store ref download state should be marked as BYPASSED"
+ )
+ self.assertEqual(ref[1],
+ "Ready",
+ "Template store ref state should be marked as Ready"
+ )
+ return
+
+ @attr(tags=["advanced", "smoke"], required_hardware="true")
+ def test_02_deploy_vm_from_direct_download_template(self):
+ """
+ Deploy a VM from a Direct Download registered template
+ """
+ bp = self.templates[0]
+ virtual_machine = VirtualMachine.create(
+ self.apiclient,
+ self.services["virtual_machine"],
+ templateid=bp.id,
+ accountid=self.account.name,
+ domainid=self.account.domainid,
+ serviceofferingid=self.service_offering.id
+ )
+ self.cleanup.append(virtual_machine)
+ return
+
+ @attr(tags=["advanced", "smoke"], required_hardware="true")
+ def test_03_deploy_vm_wrong_checksum(self):
+ """
+ Deploy a VM from a Direct Download registered template with wrong checksum
+ """
+ self.template["checksum"]="{MD5}XXXXXXX"
+ tmpl = Template.register(self.apiclient, self.template, zoneid=self.zone.id, hypervisor=self.hypervisor, randomize_name=False)
+
+ try:
+ virtual_machine = VirtualMachine.create(
+ self.apiclient,
+ self.services["virtual_machine"],
+ templateid=tmpl.id,
+ accountid=self.account.name,
+ domainid=self.account.domainid,
+ serviceofferingid=self.service_offering.id
+ )
+ self.cleanup.append(tmpl)
+ self.fail("Expected to fail deployment")
+ except Exception as e:
+ self.debug("Expected exception")
+
+ self.cleanup.append(virtual_machine)
+ self.cleanup.append(tmpl)
+ return
\ No newline at end of file
diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh
index 7f2dcd06d44..9469e8af28e 100644
--- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh
+++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh
@@ -68,7 +68,7 @@ function install_packages() {
python-flask \
haproxy \
radvd \
- sharutils genisoimage \
+ sharutils genisoimage aria2 \
strongswan libcharon-extra-plugins libstrongswan-extra-plugins \
virt-what open-vm-tools qemu-guest-agent hyperv-daemons
diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py
index a3f1043e654..02ee138aa1f 100755
--- a/tools/marvin/marvin/lib/base.py
+++ b/tools/marvin/marvin/lib/base.py
@@ -1302,6 +1302,10 @@ def register(cls, apiclient, services, zoneid=None,
if details:
cmd.details = details
+ if "directdownload" in services:
+ cmd.directdownload = services["directdownload"]
+
+
# Register Template
template = apiclient.registerTemplate(cmd)
diff --git a/ui/l10n/ar.js b/ui/l10n/ar.js
index ef30387221d..3b560a86e72 100644
--- a/ui/l10n/ar.js
+++ b/ui/l10n/ar.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "???????",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "?????? ???????? IPs",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",
diff --git a/ui/l10n/ca.js b/ui/l10n/ca.js
index 7bd14365185..f6ae80f81f3 100644
--- a/ui/l10n/ca.js
+++ b/ui/l10n/ca.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Devices",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Shared Network IPs",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",
diff --git a/ui/l10n/de_DE.js b/ui/l10n/de_DE.js
index 5bf300cffbe..934f8f98c09 100644
--- a/ui/l10n/de_DE.js
+++ b/ui/l10n/de_DE.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Ger?te",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direkt angeschlossene ?ffentliche IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Gemeinsame Netzwerk-IPs",
"label.disable.autoscale": "Automatische Skalierung deaktivieren",
"label.disable.host": "Host deaktivieren",
diff --git a/ui/l10n/en.js b/ui/l10n/en.js
index 9e2e4ae9cc9..5d951e86777 100644
--- a/ui/l10n/en.js
+++ b/ui/l10n/en.js
@@ -647,6 +647,7 @@ var dictionary = {"ICMP.code":"ICMP Code",
"label.devices":"Devices",
"label.dhcp":"DHCP",
"label.direct.attached.public.ip":"Direct Attached Public IP",
+"label.direct.download":"Direct Download",
"label.direct.ips":"Shared Network IPs",
"label.disable.autoscale":"Disable Autoscale",
"label.disable.host":"Disable Host",
diff --git a/ui/l10n/es.js b/ui/l10n/es.js
index 599f9cb0420..35257d47989 100644
--- a/ui/l10n/es.js
+++ b/ui/l10n/es.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Dispositivos",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "IP P?blica Conectada en forma Directa",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "IPs de la Red Compartida",
"label.disable.autoscale": "Deshabilitar Escalado Autom?tico",
"label.disable.host": "Deshabitar Anfitri?n",
diff --git a/ui/l10n/fr_FR.js b/ui/l10n/fr_FR.js
index 8f22fb73197..9935a818021 100644
--- a/ui/l10n/fr_FR.js
+++ b/ui/l10n/fr_FR.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Machines",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "IP publique attach?e directement",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Adresses IP du r?seau partag?",
"label.disable.autoscale": "D?sactiver Autoscale",
"label.disable.host": "D?sactiver H?te",
diff --git a/ui/l10n/hu.js b/ui/l10n/hu.js
index b1f83243f07..912ecf9a49d 100644
--- a/ui/l10n/hu.js
+++ b/ui/l10n/hu.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Eszk?z?k",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Osztott h?l?zati IP c?mek",
"label.disable.autoscale": "Automatikus sk?l?z?s kikapcsol?sa",
"label.disable.host": "Kiszolg?l? kikapcsol?sa",
diff --git a/ui/l10n/it_IT.js b/ui/l10n/it_IT.js
index 02b7e50a81f..f725584e08b 100644
--- a/ui/l10n/it_IT.js
+++ b/ui/l10n/it_IT.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Device",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Indirizzi IP di Rete condivisi",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",
diff --git a/ui/l10n/ja_JP.js b/ui/l10n/ja_JP.js
index da27a9f1d43..40cf38fd857 100644
--- a/ui/l10n/ja_JP.js
+++ b/ui/l10n/ja_JP.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "????",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "???????????????? IP ????",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "????????? IP ????",
"label.disable.autoscale": "???????????",
"label.disable.host": "???????",
diff --git a/ui/l10n/ko_KR.js b/ui/l10n/ko_KR.js
index caa38bbeca2..f6980bc067e 100644
--- a/ui/l10n/ko_KR.js
+++ b/ui/l10n/ko_KR.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "??",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "?? IP ??",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",
diff --git a/ui/l10n/nb_NO.js b/ui/l10n/nb_NO.js
index 49a8b8530cf..28cd09c3b9a 100644
--- a/ui/l10n/nb_NO.js
+++ b/ui/l10n/nb_NO.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Enheter",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direkte Tilknyttet Offentlig IP-adresse",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Deltnettverk-IPadresser",
"label.disable.autoscale": "Deaktiver autoskalering",
"label.disable.host": "Deaktiver host",
diff --git a/ui/l10n/nl_NL.js b/ui/l10n/nl_NL.js
index 3d20ff394c4..4508241ad30 100644
--- a/ui/l10n/nl_NL.js
+++ b/ui/l10n/nl_NL.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Apparaten",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "direct verbonden publieke IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Shared Netwerk IPs",
"label.disable.autoscale": "Autoscale uitschakelen",
"label.disable.host": "schakel host uit",
diff --git a/ui/l10n/pl.js b/ui/l10n/pl.js
index 7faa8c68e95..30c04b9ff75 100644
--- a/ui/l10n/pl.js
+++ b/ui/l10n/pl.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Devices",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "Direct Attached Public IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "Shared Network IPs",
"label.disable.autoscale": "Disable Autoscale",
"label.disable.host": "Disable Host",
diff --git a/ui/l10n/pt_BR.js b/ui/l10n/pt_BR.js
index a6041f3ffb6..ccfa59ae7d3 100644
--- a/ui/l10n/pt_BR.js
+++ b/ui/l10n/pt_BR.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "Dispositivos",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "IP P?blico COnectado Diretamente",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "IPs Diretos",
"label.disable.autoscale": "Desabilita Auto-escala",
"label.disable.host": "Desabilita Host",
diff --git a/ui/l10n/ru_RU.js b/ui/l10n/ru_RU.js
index 38b78d54452..6a11b38a0f7 100644
--- a/ui/l10n/ru_RU.js
+++ b/ui/l10n/ru_RU.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "??????????",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "???????? ????????? IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "?????? IP-??????",
"label.disable.autoscale": "????????? ?????????????? ??????????????",
"label.disable.host": "????????? ????",
diff --git a/ui/l10n/zh_CN.js b/ui/l10n/zh_CN.js
index 9115e94cf0d..dc44fdb9a96 100644
--- a/ui/l10n/zh_CN.js
+++ b/ui/l10n/zh_CN.js
@@ -636,6 +636,7 @@ var dictionary = {
"label.devices": "??",
"label.dhcp": "DHCP",
"label.direct.attached.public.ip": "???? IP",
+ "label.direct.download":"Direct Download",
"label.direct.ips": "???? IP",
"label.disable.autoscale": "??????",
"label.disable.host": "????",
diff --git a/ui/scripts/docs.js b/ui/scripts/docs.js
index d25dca57998..583db58bb45 100755
--- a/ui/scripts/docs.js
+++ b/ui/scripts/docs.js
@@ -1167,6 +1167,10 @@ cloudStack.docs = {
desc: 'The Management Server will download the file from the specified URL, such as http://my.web.server/filename.iso',
externalLink: ''
},
+ helpRegisterISODirectDownload: {
+ desc: 'KVM Only: Secondary Storage is bypassed and ISO is downloaded to Primary Storage on deployment',
+ externalLink: ''
+ },
helpRegisterISOZone: {
desc: 'Choose the zone where you want the ISO to be available, or All Zones to make it available throughout the cloud',
externalLink: ''
@@ -1204,6 +1208,10 @@ cloudStack.docs = {
desc: 'The Management Server will download the file from the specified URL, such as http://my.web.server/filename.vhd.gz',
externalLink: ''
},
+ helpRegisterTemplateDirectDownload: {
+ desc: 'KVM Only: Secondary Storage is bypassed and template/ISO is downloaded to Primary Storage on deployment',
+ externalLink: ''
+ },
helpRegisterTemplateZone: {
desc: 'Choose one or more zones where you want the template to be available, or All Zones to make it available throughout the cloud. (Tip: Use Ctrl to choose multiple zones)',
externalLink: ''
diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js
index c40ff29d09f..5f601d839e4 100755
--- a/ui/scripts/templates.js
+++ b/ui/scripts/templates.js
@@ -250,11 +250,13 @@
$form.find('.form-item[rel=keyboardType]').css('display', 'inline-block');
$form.find('.form-item[rel=xenserverToolsVersion61plus]').hide();
$form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide();
+ $form.find('.form-item[rel=directdownload]').hide();
} else if ($(this).val() == "XenServer") {
$form.find('.form-item[rel=rootDiskControllerType]').hide();
$form.find('.form-item[rel=nicAdapterType]').hide();
$form.find('.form-item[rel=keyboardType]').hide();
$form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide();
+ $form.find('.form-item[rel=directdownload]').hide();
if (isAdmin())
$form.find('.form-item[rel=xenserverToolsVersion61plus]').css('display', 'inline-block');
@@ -265,12 +267,14 @@
$form.find('.form-item[rel=xenserverToolsVersion61plus]').hide();
$form.find('.form-item[rel=rootDiskControllerTypeKVM]').css('display', 'inline-block');
$form.find('.form-item[rel=xenserverToolsVersion61plus]').css('display', 'inline-block');
+ $form.find('.form-item[rel=directdownload]').css('display', 'inline-block');
} else {
$form.find('.form-item[rel=rootDiskControllerType]').hide();
$form.find('.form-item[rel=nicAdapterType]').hide();
$form.find('.form-item[rel=keyboardType]').hide();
$form.find('.form-item[rel=xenserverToolsVersion61plus]').hide();
$form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide();
+ $form.find('.form-item[rel=directdownload]').hide();
}
});
@@ -285,6 +289,20 @@
}
},
+ // For KVM only: Direct Download
+ directdownload : {
+ label: 'label.direct.download',
+ docID: 'helpRegisterTemplateDirectDownload',
+ isBoolean: true,
+ dependsOn: 'hypervisor',
+ isHidden: true
+ },
+ checksum: {
+ label: 'label.checksum',
+ dependsOn: 'directdownload',
+ isHidden: true
+ },
+ // Direct Download - End
xenserverToolsVersion61plus: {
label: 'label.xenserver.tools.version.61.plus',
@@ -640,6 +658,13 @@
'details[0].rootDiskController': args.data.rootDiskControllerTypeKVM
});
}
+
+ if (args.$form.find('.form-item[rel=directdownload]').css("display") != "none" && args.data.directdownload != "") {
+ $.extend(data, {
+ 'directdownload': (args.data.directdownload == "on") ? "true" : "false",
+ 'checksum': args.data.checksum
+ });
+ }
// KVM only (ends here)
//VMware only (starts here)
@@ -1339,6 +1364,11 @@
return cloudStack.converters.convertBytes(args);
}
},
+ directdownload: {
+ label: 'label.direct.download',
+ isBoolean: true,
+ converter: cloudStack.converters.toBooleanText
+ },
isextractable: {
label: 'label.extractable.lower',
isBoolean: true,
@@ -2113,6 +2143,18 @@
required: true
}
},
+ // For KVM only: Direct Download
+ directdownload : {
+ label: 'label.direct.download',
+ docID: 'helpRegisterTemplateDirectDownload',
+ isBoolean: true
+ },
+ checksum: {
+ label: 'label.checksum',
+ dependsOn: 'directdownload',
+ isHidden: true
+ },
+ // Direct Download - End
zone: {
label: 'label.zone',
docID: 'helpRegisterISOZone',
@@ -2223,7 +2265,8 @@
url: args.data.url,
zoneid: args.data.zone,
isextractable: (args.data.isExtractable == "on"),
- bootable: (args.data.isBootable == "on")
+ bootable: (args.data.isBootable == "on"),
+ directdownload: (args.data.directdownload == "on")
};
if (args.$form.find('.form-item[rel=osTypeId]').css("display") != "none") {
@@ -2241,6 +2284,11 @@
isfeatured: (args.data.isFeatured == "on")
});
}
+ if (args.$form.find('.form-item[rel=checksum]').css("display") != "none") {
+ $.extend(data, {
+ checksum: args.data.checksum
+ });
+ }
$.ajax({
url: createURL('registerIso'),
@@ -2564,6 +2612,11 @@
required: true
}
},
+ directdownload: {
+ label: 'label.direct.download',
+ isBoolean: true,
+ converter: cloudStack.converters.toBooleanText
+ },
size: {
label: 'label.size',
converter: function(args) {
diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java
index 4d42e3cadd9..d7fbb89e47e 100644
--- a/utils/src/main/java/com/cloud/utils/UriUtils.java
+++ b/utils/src/main/java/com/cloud/utils/UriUtils.java
@@ -34,7 +34,11 @@
import java.util.List;
import java.util.ListIterator;
import java.util.StringTokenizer;
+import java.util.Map;
+import java.util.HashMap;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
@@ -55,6 +59,10 @@
import com.cloud.utils.crypt.DBEncryptionUtil;
import com.cloud.utils.exception.CloudRuntimeException;
import com.google.common.base.Strings;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
public class UriUtils {
@@ -289,6 +297,94 @@ public static long getRemoteSize(String url) {
}
}
+ protected static Map<String, List<String>> getMultipleValuesFromXML(InputStream is, String[] tagNames) {
+ Map<String, List<String>> returnValues = new HashMap<String, List<String>>();
+ try {
+ DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder docBuilder = factory.newDocumentBuilder();
+ Document doc = docBuilder.parse(is);
+ Element rootElement = doc.getDocumentElement();
+ for (int i = 0; i < tagNames.length; i++) {
+ NodeList targetNodes = rootElement.getElementsByTagName(tagNames[i]);
+ if (targetNodes.getLength() <= 0) {
+ s_logger.error("no " + tagNames[i] + " tag in XML response...returning null");
+ } else {
+ List<String> valueList = new ArrayList<String>();
+ for (int j = 0; j < targetNodes.getLength(); j++) {
+ Node node = targetNodes.item(j);
+ valueList.add(node.getTextContent());
+ }
+ returnValues.put(tagNames[i], valueList);
+ }
+ }
+ } catch (Exception ex) {
+ s_logger.error(ex);
+ }
+ return returnValues;
+ }
+
+ /**
+ * Check if there is at least one existent URL defined on metalink
+ * @param url metalink url
+ * @return true if at least one existent URL defined on metalink, false if not
+ */
+ protected static boolean checkUrlExistenceMetalink(String url) {
+ HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager());
+ GetMethod getMethod = new GetMethod(url);
+ try {
+ if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
+ InputStream is = getMethod.getResponseBodyAsStream();
+ Map<String, List<String>> metalinkUrls = getMultipleValuesFromXML(is, new String[] {"url"});
+ if (metalinkUrls.containsKey("url")) {
+ List<String> urls = metalinkUrls.get("url");
+ boolean validUrl = false;
+ for (String u : urls) {
+ if (url.endsWith("torrent")) {
+ continue;
+ }
+ try {
+ UriUtils.checkUrlExistence(u);
+ validUrl = true;
+ break;
+ }
+ catch (IllegalArgumentException e) {
+ s_logger.warn(e.getMessage());
+ }
+ }
+ return validUrl;
+ }
+ }
+ } catch (IOException e) {
+ s_logger.warn(e.getMessage());
+ }
+ return false;
+ }
+
+ /**
+ * Get list of urls on metalink
+ * @param metalinkUrl
+ * @return
+ */
+ public static List<String> getMetalinkUrls(String metalinkUrl) {
+ HttpClient httpClient = new HttpClient(new MultiThreadedHttpConnectionManager());
+ GetMethod getMethod = new GetMethod(metalinkUrl);
+ List<String> urls = new ArrayList<>();
+ try (
+ InputStream is = getMethod.getResponseBodyAsStream()
+ ) {
+ if (httpClient.executeMethod(getMethod) == HttpStatus.SC_OK) {
+ Map<String, List<String>> metalinkUrlsMap = getMultipleValuesFromXML(is, new String[] {"url"});
+ if (metalinkUrlsMap.containsKey("url")) {
+ List<String> metalinkUrls = metalinkUrlsMap.get("url");
+ urls.addAll(metalinkUrls);
+ }
+ }
+ } catch (IOException e) {
+ s_logger.warn(e.getMessage());
+ }
+ return urls;
+ }
+
// use http HEAD method to validate url
public static void checkUrlExistence(String url) {
if (url.toLowerCase().startsWith("http") || url.toLowerCase().startsWith("https")) {
@@ -298,6 +394,9 @@ public static void checkUrlExistence(String url) {
if (httpClient.executeMethod(httphead) != HttpStatus.SC_OK) {
throw new IllegalArgumentException("Invalid URL: " + url);
}
+ if (url.endsWith("metalink") && !checkUrlExistenceMetalink(url)) {
+ throw new IllegalArgumentException("Invalid URLs defined on metalink: " + url);
+ }
} catch (HttpException hte) {
throw new IllegalArgumentException("Cannot reach URL: " + url);
} catch (IOException ioe) {
@@ -320,7 +419,8 @@ private static void checkFormat(String format, String uripath) {
(!uripath.toLowerCase().endsWith("img.gz")) && (!uripath.toLowerCase().endsWith("img.zip")) && (!uripath.toLowerCase().endsWith("img.bz2")) &&
(!uripath.toLowerCase().endsWith("raw")) && (!uripath.toLowerCase().endsWith("raw.gz")) && (!uripath.toLowerCase().endsWith("raw.bz2")) &&
(!uripath.toLowerCase().endsWith("raw.zip")) && (!uripath.toLowerCase().endsWith("iso")) && (!uripath.toLowerCase().endsWith("iso.zip"))
- && (!uripath.toLowerCase().endsWith("iso.bz2")) && (!uripath.toLowerCase().endsWith("iso.gz"))) {
+ && (!uripath.toLowerCase().endsWith("iso.bz2")) && (!uripath.toLowerCase().endsWith("iso.gz"))
+ && (!uripath.toLowerCase().endsWith("metalink"))) {
throw new IllegalArgumentException("Please specify a valid " + format.toLowerCase());
}
@@ -338,7 +438,8 @@ private static void checkFormat(String format, String uripath) {
&& (!uripath.toLowerCase().endsWith("qcow2")
&& !uripath.toLowerCase().endsWith("qcow2.zip")
&& !uripath.toLowerCase().endsWith("qcow2.bz2")
- && !uripath.toLowerCase().endsWith("qcow2.gz")))
+ && !uripath.toLowerCase().endsWith("qcow2.gz"))
+ && !uripath.toLowerCase().endsWith("metalink"))
|| (format.equalsIgnoreCase("ova")
&& (!uripath.toLowerCase().endsWith("ova")
&& !uripath.toLowerCase().endsWith("ova.zip")
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services