You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ah...@apache.org on 2014/02/05 02:39:43 UTC
[05/11] Moved the secondary storage service into its own server
directory
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java
new file mode 100755
index 0000000..ee50647
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageDiscoverer.java
@@ -0,0 +1,312 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.resource;
+
+import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import javax.ejb.Local;
+import javax.inject.Inject;
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.host.HostVO;
+import com.cloud.host.Status.Event;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.resource.Discoverer;
+import com.cloud.resource.DiscovererBase;
+import com.cloud.resource.ServerResource;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.VMTemplateZoneVO;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VMTemplateZoneDao;
+import com.cloud.storage.resource.DummySecondaryStorageResource;
+import com.cloud.utils.component.ComponentContext;
+import com.cloud.utils.net.NfsUtils;
+import com.cloud.utils.script.Script;
+
+/**
+ * SecondaryStorageDiscoverer is used to discover secondary
+ * storage servers and make sure everything it can do is
+ * correct.
+ */
+@Local(value = Discoverer.class)
+public class SecondaryStorageDiscoverer extends DiscovererBase implements Discoverer {
+ private static final Logger s_logger = Logger.getLogger(SecondaryStorageDiscoverer.class);
+
+ long _timeout = 2 * 60 * 1000; // 2 minutes
+ String _mountParent;
+ boolean _useServiceVM = false;
+
+ Random _random = new Random(System.currentTimeMillis());
+ @Inject
+ protected VMTemplateDao _tmpltDao = null;
+ @Inject
+ protected VMTemplateZoneDao _vmTemplateZoneDao = null;
+ @Inject
+ protected VMTemplateDao _vmTemplateDao = null;
+ @Inject
+ protected AgentManager _agentMgr = null;
+
+ protected SecondaryStorageDiscoverer() {
+ }
+
+ @Override
+ public Map<? extends ServerResource, Map<String, String>>
+ find(long dcId, Long podId, Long clusterId, URI uri, String username, String password, List<String> hostTags) {
+ if (!uri.getScheme().equalsIgnoreCase("nfs") && !uri.getScheme().equalsIgnoreCase("cifs") && !uri.getScheme().equalsIgnoreCase("file") &&
+ !uri.getScheme().equalsIgnoreCase("iso") && !uri.getScheme().equalsIgnoreCase("dummy")) {
+ s_logger.debug("It's not NFS or file or ISO, so not a secondary storage server: " + uri.toString());
+ return null;
+ }
+
+ if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs") || uri.getScheme().equalsIgnoreCase("iso")) {
+ return createNfsSecondaryStorageResource(dcId, podId, uri);
+ } else if (uri.getScheme().equalsIgnoreCase("file")) {
+ return createLocalSecondaryStorageResource(dcId, podId, uri);
+ } else if (uri.getScheme().equalsIgnoreCase("dummy")) {
+ return createDummySecondaryStorageResource(dcId, podId, uri);
+ } else {
+ return null;
+ }
+ }
+
+ protected Map<? extends ServerResource, Map<String, String>> createNfsSecondaryStorageResource(long dcId, Long podId, URI uri) {
+
+ if (_useServiceVM) {
+ return createDummySecondaryStorageResource(dcId, podId, uri);
+ }
+ String mountStr = NfsUtils.uri2Mount(uri);
+
+ Script script = new Script(true, "mount", _timeout, s_logger);
+ String mntPoint = null;
+ File file = null;
+ do {
+ mntPoint = _mountParent + File.separator + Integer.toHexString(_random.nextInt(Integer.MAX_VALUE));
+ file = new File(mntPoint);
+ } while (file.exists());
+
+ if (!file.mkdirs()) {
+ s_logger.warn("Unable to make directory: " + mntPoint);
+ return null;
+ }
+
+ script.add(mountStr, mntPoint);
+ String result = script.execute();
+ if (result != null && !result.contains("already mounted")) {
+ s_logger.warn("Unable to mount " + uri.toString() + " due to " + result);
+ file.delete();
+ return null;
+ }
+
+ script = new Script(true, "umount", 0, s_logger);
+ script.add(mntPoint);
+ script.execute();
+
+ file.delete();
+
+ Map<NfsSecondaryStorageResource, Map<String, String>> srs = new HashMap<NfsSecondaryStorageResource, Map<String, String>>();
+
+ NfsSecondaryStorageResource storage;
+ if (_configDao.isPremium()) {
+ Class<?> impl;
+ String name = "com.cloud.storage.resource.PremiumSecondaryStorageResource";
+ try {
+ impl = Class.forName(name);
+ final Constructor<?> constructor = impl.getDeclaredConstructor();
+ constructor.setAccessible(true);
+ storage = (NfsSecondaryStorageResource)constructor.newInstance();
+ } catch (final ClassNotFoundException e) {
+ s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to ClassNotFoundException");
+ return null;
+ } catch (final SecurityException e) {
+ s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to SecurityException");
+ return null;
+ } catch (final NoSuchMethodException e) {
+ s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to NoSuchMethodException");
+ return null;
+ } catch (final IllegalArgumentException e) {
+ s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to IllegalArgumentException");
+ return null;
+ } catch (final InstantiationException e) {
+ s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to InstantiationException");
+ return null;
+ } catch (final IllegalAccessException e) {
+ s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to IllegalAccessException");
+ return null;
+ } catch (final InvocationTargetException e) {
+ s_logger.error("Unable to load com.cloud.storage.resource.PremiumSecondaryStorageResource due to InvocationTargetException");
+ return null;
+ }
+ } else {
+ storage = new NfsSecondaryStorageResource();
+ }
+
+ Map<String, String> details = new HashMap<String, String>();
+ details.put("mount.path", mountStr);
+ details.put("orig.url", uri.toString());
+ details.put("mount.parent", _mountParent);
+
+ Map<String, Object> params = new HashMap<String, Object>();
+ params.putAll(details);
+ params.put("zone", Long.toString(dcId));
+ if (podId != null) {
+ params.put("pod", podId.toString());
+ }
+ params.put("guid", uri.toString());
+ params.put("secondary.storage.vm", "false");
+ params.put("max.template.iso.size", _configDao.getValue("max.template.iso.size"));
+
+ try {
+ storage.configure("Storage", params);
+ } catch (ConfigurationException e) {
+ s_logger.warn("Unable to configure the storage ", e);
+ return null;
+ }
+ srs.put(storage, details);
+
+ return srs;
+ }
+
+ protected Map<? extends ServerResource, Map<String, String>> createLocalSecondaryStorageResource(long dcId, Long podId, URI uri) {
+ Map<LocalSecondaryStorageResource, Map<String, String>> srs = new HashMap<LocalSecondaryStorageResource, Map<String, String>>();
+
+ LocalSecondaryStorageResource storage = new LocalSecondaryStorageResource();
+ storage = ComponentContext.inject(storage);
+
+ Map<String, String> details = new HashMap<String, String>();
+
+ File file = new File(uri);
+ details.put("mount.path", file.getAbsolutePath());
+ details.put("orig.url", uri.toString());
+
+ Map<String, Object> params = new HashMap<String, Object>();
+ params.putAll(details);
+ params.put("zone", Long.toString(dcId));
+ if (podId != null) {
+ params.put("pod", podId.toString());
+ }
+ params.put("guid", uri.toString());
+
+ try {
+ storage.configure("Storage", params);
+ } catch (ConfigurationException e) {
+ s_logger.warn("Unable to configure the storage ", e);
+ return null;
+ }
+ srs.put(storage, details);
+
+ return srs;
+ }
+
+ protected Map<ServerResource, Map<String, String>> createDummySecondaryStorageResource(long dcId, Long podId, URI uri) {
+ Map<ServerResource, Map<String, String>> srs = new HashMap<ServerResource, Map<String, String>>();
+
+ DummySecondaryStorageResource storage = new DummySecondaryStorageResource(_useServiceVM);
+ storage = ComponentContext.inject(storage);
+
+ Map<String, String> details = new HashMap<String, String>();
+
+ details.put("mount.path", uri.toString());
+ details.put("orig.url", uri.toString());
+
+ Map<String, Object> params = new HashMap<String, Object>();
+ params.putAll(details);
+ params.put("zone", Long.toString(dcId));
+ if (podId != null) {
+ params.put("pod", podId.toString());
+ }
+ params.put("guid", uri.toString());
+
+ try {
+ storage.configure("Storage", params);
+ } catch (ConfigurationException e) {
+ s_logger.warn("Unable to configure the storage ", e);
+ return null;
+ }
+ srs.put(storage, details);
+
+ return srs;
+ }
+
+ @Override
+ public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+ super.configure(name, params);
+
+ _mountParent = _params.get("mount.parent");
+ if (_mountParent == null) {
+ _mountParent = "/mnt";
+ }
+
+ String useServiceVM = _params.get("secondary.storage.vm");
+ if ("true".equalsIgnoreCase(useServiceVM)) {
+ _useServiceVM = true;
+ }
+ return true;
+ }
+
+ @Override
+ public boolean matchHypervisor(String hypervisor) {
+ if (hypervisor.equals("SecondaryStorage")) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public Hypervisor.HypervisorType getHypervisorType() {
+ return Hypervisor.HypervisorType.None;
+ }
+
+ @Override
+ public void postDiscovery(List<HostVO> hosts, long msId) {
+ if (_useServiceVM) {
+ for (HostVO h : hosts) {
+ _agentMgr.agentStatusTransitTo(h, Event.AgentDisconnected, msId);
+ }
+ }
+ for (HostVO h : hosts) {
+ associateTemplatesToZone(h.getId(), h.getDataCenterId());
+ }
+
+ }
+
+ private void associateTemplatesToZone(long hostId, long dcId) {
+ VMTemplateZoneVO tmpltZone;
+
+ List<VMTemplateVO> allTemplates = _vmTemplateDao.listAll();
+ for (VMTemplateVO vt : allTemplates) {
+ if (vt.isCrossZones()) {
+ tmpltZone = _vmTemplateZoneDao.findByZoneTemplate(dcId, vt.getId());
+ if (tmpltZone == null) {
+ VMTemplateZoneVO vmTemplateZone = new VMTemplateZoneVO(dcId, vt.getId(), new Date());
+ _vmTemplateZoneDao.persist(vmTemplateZone);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java
new file mode 100755
index 0000000..93fd8ea
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResource.java
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.resource;
+
+import com.cloud.resource.ServerResource;
+
+/**
+ *
+ * SecondaryStorageServerResource is a generic container to execute commands sent
+ */
+public interface SecondaryStorageResource extends ServerResource {
+
+ public String getRootDir(String cmd);
+
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java
new file mode 100644
index 0000000..14ebc71
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/SecondaryStorageResourceHandler.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.resource;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+
+public interface SecondaryStorageResourceHandler {
+ Answer executeRequest(Command cmd);
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
new file mode 100644
index 0000000..d0abe2c
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManager.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import java.util.Map;
+
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
+import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.Proxy;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.template.TemplateDownloader;
+import com.cloud.storage.template.TemplateProp;
+import com.cloud.utils.component.Manager;
+
+public interface DownloadManager extends Manager {
+
+ /**
+ * Initiate download of a public template
+ * @param id unique id.
+ * @param url the url from where to download from
+ * @param hvm whether the template is a hardware virtual machine
+ * @param accountId the accountId of the iso owner (null if public iso)
+ * @param descr description of the template
+ * @param user username used for authentication to the server
+ * @param password password used for authentication to the server
+ * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes.
+ * @param resourceType signifying the type of resource like template, volume etc.
+ * @return job-id that can be used to interrogate the status of the download.
+ */
+ public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+ String installPathPrefix, String templatePath, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType);
+
+ public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+ String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType);
+
+ /**
+ * Get the status of a download job
+ * @param jobId job Id
+ * @return status of the download job
+ */
+ public TemplateDownloader.Status getDownloadStatus(String jobId);
+
+ /**
+ * Get the status of a download job
+ * @param jobId job Id
+ * @return status of the download job
+ */
+ public VMTemplateHostVO.Status getDownloadStatus2(String jobId);
+
+ /**
+ * Get the download percent of a download job
+ * @param jobId job Id
+ * @return
+ */
+ public int getDownloadPct(String jobId);
+
+ /**
+ * Get the download error if any
+ * @param jobId job Id
+ * @return
+ */
+ public String getDownloadError(String jobId);
+
+ /**
+ * Get the local path for the download
+ * @param jobId job Id
+ * @return
+ public String getDownloadLocalPath(String jobId);
+ */
+
+ /** Handle download commands from the management server
+ * @param cmd cmd from server
+ * @return answer representing status of download.
+ */
+ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd);
+
+ /**
+ /**
+ * @return list of template info for installed templates
+ */
+ public Map<String, TemplateProp> gatherTemplateInfo(String templateDir);
+
+ /**
+ /**
+ * @return list of volume info for installed volumes
+ */
+ public Map<Long, TemplateProp> gatherVolumeInfo(String volumeDir);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
new file mode 100755
index 0000000..d45a6bb
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -0,0 +1,1080 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigInteger;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.ejb.Local;
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
+import org.apache.cloudstack.storage.command.DownloadProgressCommand;
+import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType;
+import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.Proxy;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.exception.InternalErrorException;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.StorageLayer;
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.template.HttpTemplateDownloader;
+import com.cloud.storage.template.IsoProcessor;
+import com.cloud.storage.template.LocalTemplateDownloader;
+import com.cloud.storage.template.OVAProcessor;
+import com.cloud.storage.template.Processor;
+import com.cloud.storage.template.Processor.FormatInfo;
+import com.cloud.storage.template.QCOW2Processor;
+import com.cloud.storage.template.RawImageProcessor;
+import com.cloud.storage.template.S3TemplateDownloader;
+import com.cloud.storage.template.ScpTemplateDownloader;
+import com.cloud.storage.template.TemplateConstants;
+import com.cloud.storage.template.TemplateDownloader;
+import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
+import com.cloud.storage.template.TemplateDownloader.Status;
+import com.cloud.storage.template.TemplateLocation;
+import com.cloud.storage.template.TemplateProp;
+import com.cloud.storage.template.VhdProcessor;
+import com.cloud.storage.template.VmdkProcessor;
+import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.OutputInterpreter;
+import com.cloud.utils.script.Script;
+
+@Local(value = DownloadManager.class)
+public class DownloadManagerImpl extends ManagerBase implements DownloadManager {
+ private String _name;
+ StorageLayer _storage;
+ Map<String, Processor> _processors;
+
+ public class Completion implements DownloadCompleteCallback {
+ private final String jobId;
+
+ public Completion(String jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public void downloadComplete(Status status) {
+ setDownloadStatus(jobId, status);
+ }
+ }
+
+ private static class DownloadJob {
+ private final TemplateDownloader td;
+ private final String tmpltName;
+ private final boolean hvm;
+ private final ImageFormat format;
+ private String tmpltPath;
+ private final String description;
+ private String checksum;
+ private final String installPathPrefix;
+ private long templatesize;
+ private long templatePhysicalSize;
+ private final long id;
+ private final ResourceType resourceType;
+
+ public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+ String installPathPrefix, ResourceType resourceType) {
+ super();
+ this.td = td;
+ this.tmpltName = tmpltName;
+ this.format = format;
+ this.hvm = hvm;
+ description = descr;
+ checksum = cksum;
+ this.installPathPrefix = installPathPrefix;
+ templatesize = 0;
+ this.id = id;
+ this.resourceType = resourceType;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public String getChecksum() {
+ return checksum;
+ }
+
+ public TemplateDownloader getTemplateDownloader() {
+ return td;
+ }
+
+ public String getTmpltName() {
+ return tmpltName;
+ }
+
+ public ImageFormat getFormat() {
+ return format;
+ }
+
+ public boolean isHvm() {
+ return hvm;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public ResourceType getResourceType() {
+ return resourceType;
+ }
+
+ public void setTmpltPath(String tmpltPath) {
+ this.tmpltPath = tmpltPath;
+ }
+
+ public String getTmpltPath() {
+ return tmpltPath;
+ }
+
+ public String getInstallPathPrefix() {
+ return installPathPrefix;
+ }
+
+ public void cleanup() {
+ if (td != null) {
+ String dnldPath = td.getDownloadLocalPath();
+ if (dnldPath != null) {
+ File f = new File(dnldPath);
+ File dir = f.getParentFile();
+ f.delete();
+ if (dir != null) {
+ dir.delete();
+ }
+ }
+ }
+
+ }
+
+ public void setTemplatesize(long templatesize) {
+ this.templatesize = templatesize;
+ }
+
+ public long getTemplatesize() {
+ return templatesize;
+ }
+
+ public void setTemplatePhysicalSize(long templatePhysicalSize) {
+ this.templatePhysicalSize = templatePhysicalSize;
+ }
+
+ public long getTemplatePhysicalSize() {
+ return templatePhysicalSize;
+ }
+
+ public void setCheckSum(String checksum) {
+ this.checksum = checksum;
+ }
+ }
+
+ public static final Logger s_logger = Logger.getLogger(DownloadManagerImpl.class);
+ private String _templateDir;
+ private String _volumeDir;
+ private String createTmpltScr;
+ private String createVolScr;
+
+ private ExecutorService threadPool;
+
+ private final Map<String, DownloadJob> jobs = new ConcurrentHashMap<String, DownloadJob>();
+ private String listTmpltScr;
+ private String listVolScr;
+ private int installTimeoutPerGig = 180 * 60 * 1000;
+
+ public void setThreadPool(ExecutorService threadPool) {
+ this.threadPool = threadPool;
+ }
+
+ public void setStorageLayer(StorageLayer storage) {
+ _storage = storage;
+ }
+
+ /**
+ * Get notified of change of job status. Executed in context of downloader
+ * thread
+ *
+ * @param jobId
+ * the id of the job
+ * @param status
+ * the status of the job
+ */
+ public void setDownloadStatus(String jobId, Status status) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj == null) {
+ s_logger.warn("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found");
+ return;
+ }
+ TemplateDownloader td = dj.getTemplateDownloader();
+ s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status);
+ s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" +
+ td.getDownloadPercent());
+
+ switch (status) {
+ case ABORTED:
+ case NOT_STARTED:
+ case UNRECOVERABLE_ERROR:
+ // TODO
+ dj.cleanup();
+ break;
+ case UNKNOWN:
+ return;
+ case IN_PROGRESS:
+ s_logger.info("Resuming jobId: " + jobId + ", status=" + status);
+ td.setResume(true);
+ threadPool.execute(td);
+ break;
+ case RECOVERABLE_ERROR:
+ threadPool.execute(td);
+ break;
+ case DOWNLOAD_FINISHED:
+ if (!(td instanceof S3TemplateDownloader)) {
+ // we currently only create template.properties for NFS by
+ // running some post download script
+ td.setDownloadError("Download success, starting install ");
+ String result = postDownload(jobId);
+ if (result != null) {
+ s_logger.error("Failed post download script: " + result);
+ td.setStatus(Status.UNRECOVERABLE_ERROR);
+ td.setDownloadError("Failed post download script: " + result);
+ } else {
+ td.setStatus(Status.POST_DOWNLOAD_FINISHED);
+ td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
+ }
+ } else {
+ // for s3 and swift, we skip post download step and just set
+ // status to trigger callback.
+ td.setStatus(Status.POST_DOWNLOAD_FINISHED);
+ // set template size for S3
+ S3TemplateDownloader std = (S3TemplateDownloader)td;
+ long size = std.totalBytes;
+ DownloadJob dnld = jobs.get(jobId);
+ dnld.setTemplatesize(size);
+ dnld.setTemplatePhysicalSize(size);
+ dnld.setTmpltPath(std.getDownloadLocalPath()); // update template path to include file name.
+ }
+ dj.cleanup();
+ break;
+ default:
+ break;
+ }
+ }
+
+ private String computeCheckSum(File f) {
+ byte[] buffer = new byte[8192];
+ int read = 0;
+ MessageDigest digest;
+ String checksum = null;
+ InputStream is = null;
+ try {
+ digest = MessageDigest.getInstance("MD5");
+ is = new FileInputStream(f);
+ while ((read = is.read(buffer)) > 0) {
+ digest.update(buffer, 0, read);
+ }
+ byte[] md5sum = digest.digest();
+ BigInteger bigInt = new BigInteger(1, md5sum);
+ checksum = String.format("%032x", bigInt);
+ return checksum;
+ } catch (IOException e) {
+ return null;
+ } catch (NoSuchAlgorithmException e) {
+ return null;
+ } finally {
+ try {
+ if (is != null)
+ is.close();
+ } catch (IOException e) {
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Post download activity (install and cleanup). Executed in context of
+ * downloader thread
+ *
+ * @throws IOException
+ */
+ private String postDownload(String jobId) {
+ DownloadJob dnld = jobs.get(jobId);
+ TemplateDownloader td = dnld.getTemplateDownloader();
+ String resourcePath = dnld.getInstallPathPrefix(); // path with mount
+ // directory
+ String finalResourcePath = dnld.getTmpltPath(); // template download
+ // path on secondary
+ // storage
+ ResourceType resourceType = dnld.getResourceType();
+
+ File originalTemplate = new File(td.getDownloadLocalPath());
+ String checkSum = computeCheckSum(originalTemplate);
+ if (checkSum == null) {
+ s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!");
+ }
+ dnld.setCheckSum(checkSum);
+
+ int imgSizeGigs = (int)Math.ceil(_storage.getSize(td.getDownloadLocalPath()) * 1.0d / (1024 * 1024 * 1024));
+ imgSizeGigs++; // add one just in case
+ long timeout = imgSizeGigs * installTimeoutPerGig;
+ Script scr = null;
+ String script = resourceType == ResourceType.TEMPLATE ? createTmpltScr : createVolScr;
+ scr = new Script(script, timeout, s_logger);
+ scr.add("-s", Integer.toString(imgSizeGigs));
+ scr.add("-S", Long.toString(td.getMaxTemplateSizeInBytes()));
+ if (dnld.getDescription() != null && dnld.getDescription().length() > 1) {
+ scr.add("-d", dnld.getDescription());
+ }
+ if (dnld.isHvm()) {
+ scr.add("-h");
+ }
+
+ // add options common to ISO and template
+ String extension = dnld.getFormat().getFileExtension();
+ String templateName = "";
+ if (extension.equals("iso")) {
+ templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_");
+ } else {
+ templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes()).toString();
+ }
+
+ // run script to mv the temporary template file to the final template
+ // file
+ String templateFilename = templateName + "." + extension;
+ dnld.setTmpltPath(finalResourcePath + "/" + templateFilename);
+ scr.add("-n", templateFilename);
+
+ scr.add("-t", resourcePath);
+ scr.add("-f", td.getDownloadLocalPath()); // this is the temporary
+ // template file downloaded
+ if (dnld.getChecksum() != null && dnld.getChecksum().length() > 1) {
+ scr.add("-c", dnld.getChecksum());
+ }
+ scr.add("-u"); // cleanup
+ String result;
+ result = scr.execute();
+
+ if (result != null) {
+ return result;
+ }
+
+ // Set permissions for the downloaded template
+ File downloadedTemplate = new File(resourcePath + "/" + templateFilename);
+ _storage.setWorldReadableAndWriteable(downloadedTemplate);
+
+ // Set permissions for template/volume.properties
+ String propertiesFile = resourcePath;
+ if (resourceType == ResourceType.TEMPLATE) {
+ propertiesFile += "/template.properties";
+ } else {
+ propertiesFile += "/volume.properties";
+ }
+ File templateProperties = new File(propertiesFile);
+ _storage.setWorldReadableAndWriteable(templateProperties);
+
+ TemplateLocation loc = new TemplateLocation(_storage, resourcePath);
+ try {
+ loc.create(dnld.getId(), true, dnld.getTmpltName());
+ } catch (IOException e) {
+ s_logger.warn("Something is wrong with template location " + resourcePath, e);
+ loc.purge();
+ return "Unable to download due to " + e.getMessage();
+ }
+
+ Iterator<Processor> en = _processors.values().iterator();
+ while (en.hasNext()) {
+ Processor processor = en.next();
+
+ FormatInfo info = null;
+ try {
+ info = processor.process(resourcePath, null, templateName);
+ } catch (InternalErrorException e) {
+ s_logger.error("Template process exception ", e);
+ return e.toString();
+ }
+ if (info != null) {
+ loc.addFormat(info);
+ dnld.setTemplatesize(info.virtualSize);
+ dnld.setTemplatePhysicalSize(info.size);
+ break;
+ }
+ }
+
+ if (!loc.save()) {
+ s_logger.warn("Cleaning up because we're unable to save the formats");
+ loc.purge();
+ }
+
+ return null;
+ }
+
+ @Override
+ public Status getDownloadStatus(String jobId) {
+ DownloadJob job = jobs.get(jobId);
+ if (job != null) {
+ TemplateDownloader td = job.getTemplateDownloader();
+ if (td != null) {
+ return td.getStatus();
+ }
+ }
+ return Status.UNKNOWN;
+ }
+
+ @Override
+ public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+ String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+ UUID uuid = UUID.randomUUID();
+ String jobId = uuid.toString();
+
+ URI uri;
+ try {
+ uri = new URI(url);
+ } catch (URISyntaxException e) {
+ throw new CloudRuntimeException("URI is incorrect: " + url);
+ }
+ TemplateDownloader td;
+ if ((uri != null) && (uri.getScheme() != null)) {
+ if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
+ td = new S3TemplateDownloader(s3, url, installPathPrefix, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
+ } else {
+ throw new CloudRuntimeException("Scheme is not supported " + url);
+ }
+ } else {
+ throw new CloudRuntimeException("Unable to download from URL: " + url);
+ }
+ DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
+ dj.setTmpltPath(installPathPrefix);
+ jobs.put(jobId, dj);
+ threadPool.execute(td);
+
+ return jobId;
+ }
+
+ @Override
+ public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+ String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
+ UUID uuid = UUID.randomUUID();
+ String jobId = uuid.toString();
+ String tmpDir = installPathPrefix;
+
+ try {
+
+ if (!_storage.mkdirs(tmpDir)) {
+ s_logger.warn("Unable to create " + tmpDir);
+ return "Unable to create " + tmpDir;
+ }
+ // TO DO - define constant for volume properties.
+ File file =
+ ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator +
+ "volume.properties");
+ if (file.exists()) {
+ file.delete();
+ }
+
+ if (!file.createNewFile()) {
+ s_logger.warn("Unable to create new file: " + file.getAbsolutePath());
+ return "Unable to create new file: " + file.getAbsolutePath();
+ }
+
+ URI uri;
+ try {
+ uri = new URI(url);
+ } catch (URISyntaxException e) {
+ throw new CloudRuntimeException("URI is incorrect: " + url);
+ }
+ TemplateDownloader td;
+ if ((uri != null) && (uri.getScheme() != null)) {
+ if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
+ td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
+ } else if (uri.getScheme().equalsIgnoreCase("file")) {
+ td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
+ } else if (uri.getScheme().equalsIgnoreCase("scp")) {
+ td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
+ } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) {
+ td = null;
+ // TODO: implement this.
+ throw new CloudRuntimeException("Scheme is not supported " + url);
+ } else {
+ throw new CloudRuntimeException("Scheme is not supported " + url);
+ }
+ } else {
+ throw new CloudRuntimeException("Unable to download from URL: " + url);
+ }
+ // NOTE the difference between installPathPrefix and templatePath
+ // here. instalPathPrefix is the absolute path for template
+ // including mount directory
+ // on ssvm, while templatePath is the final relative path on
+ // secondary storage.
+ DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
+ dj.setTmpltPath(templatePath);
+ jobs.put(jobId, dj);
+ threadPool.execute(td);
+
+ return jobId;
+ } catch (IOException e) {
+ s_logger.warn("Unable to download to " + tmpDir, e);
+ return null;
+ }
+ }
+
+ @Override
+ public String getDownloadError(String jobId) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj != null) {
+ return dj.getTemplateDownloader().getDownloadError();
+ }
+ return null;
+ }
+
+ public long getDownloadTemplateSize(String jobId) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj != null) {
+ return dj.getTemplatesize();
+ }
+ return 0;
+ }
+
+ public String getDownloadCheckSum(String jobId) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj != null) {
+ return dj.getChecksum();
+ }
+ return null;
+ }
+
+ public long getDownloadTemplatePhysicalSize(String jobId) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj != null) {
+ return dj.getTemplatePhysicalSize();
+ }
+ return 0;
+ }
+
+ // @Override
+ public String getDownloadLocalPath(String jobId) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj != null) {
+ return dj.getTemplateDownloader().getDownloadLocalPath();
+ }
+ return null;
+ }
+
+ @Override
+ public int getDownloadPct(String jobId) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj != null) {
+ return dj.getTemplateDownloader().getDownloadPercent();
+ }
+ return 0;
+ }
+
+ public static VMTemplateHostVO.Status convertStatus(Status tds) {
+ switch (tds) {
+ case ABORTED:
+ return VMTemplateHostVO.Status.NOT_DOWNLOADED;
+ case DOWNLOAD_FINISHED:
+ return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS;
+ case IN_PROGRESS:
+ return VMTemplateHostVO.Status.DOWNLOAD_IN_PROGRESS;
+ case NOT_STARTED:
+ return VMTemplateHostVO.Status.NOT_DOWNLOADED;
+ case RECOVERABLE_ERROR:
+ return VMTemplateHostVO.Status.NOT_DOWNLOADED;
+ case UNKNOWN:
+ return VMTemplateHostVO.Status.UNKNOWN;
+ case UNRECOVERABLE_ERROR:
+ return VMTemplateHostVO.Status.DOWNLOAD_ERROR;
+ case POST_DOWNLOAD_FINISHED:
+ return VMTemplateHostVO.Status.DOWNLOADED;
+ default:
+ return VMTemplateHostVO.Status.UNKNOWN;
+ }
+ }
+
+ @Override
+ public com.cloud.storage.VMTemplateHostVO.Status getDownloadStatus2(String jobId) {
+ return convertStatus(getDownloadStatus(jobId));
+ }
+
+ @Override
+ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) {
+ ResourceType resourceType = cmd.getResourceType();
+ if (cmd instanceof DownloadProgressCommand) {
+ return handleDownloadProgressCmd(resource, (DownloadProgressCommand)cmd);
+ }
+
+ if (cmd.getUrl() == null) {
+ return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download",
+ VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+ }
+
+ if (cmd.getName() == null) {
+ return new DownloadAnswer("Invalid Name", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+ }
+
+ DataStoreTO dstore = cmd.getDataStore();
+ String installPathPrefix = cmd.getInstallPath();
+ // for NFS, we need to get mounted path
+ if (dstore instanceof NfsTO) {
+ installPathPrefix = resource.getRootDir(((NfsTO)dstore).getUrl()) + File.separator + installPathPrefix;
+ }
+ String user = null;
+ String password = null;
+ if (cmd.getAuth() != null) {
+ user = cmd.getAuth().getUserName();
+ password = cmd.getAuth().getPassword();
+ }
+ // TO DO - Define Volume max size as well
+ long maxDownloadSizeInBytes =
+ (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes());
+ String jobId = null;
+ if (dstore instanceof S3TO) {
+ jobId =
+ downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
+ cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+ } else {
+ jobId =
+ downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(),
+ cmd.getChecksum(), installPathPrefix, cmd.getInstallPath(), user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
+ }
+ sleep();
+ if (jobId == null) {
+ return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
+ }
+ return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId),
+ getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId));
+ }
+
+ private void sleep() {
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+
+ private DownloadAnswer handleDownloadProgressCmd(SecondaryStorageResource resource, DownloadProgressCommand cmd) {
+ String jobId = cmd.getJobId();
+ DownloadAnswer answer;
+ DownloadJob dj = null;
+ if (jobId != null) {
+ dj = jobs.get(jobId);
+ }
+ if (dj == null) {
+ if (cmd.getRequest() == RequestType.GET_OR_RESTART) {
+ DownloadCommand dcmd = new DownloadCommand(cmd);
+ return handleDownloadCommand(resource, dcmd);
+ } else {
+ return new DownloadAnswer("Cannot find job", com.cloud.storage.VMTemplateStorageResourceAssoc.Status.UNKNOWN);
+ }
+ }
+ TemplateDownloader td = dj.getTemplateDownloader();
+ switch (cmd.getRequest()) {
+ case GET_STATUS:
+ break;
+ case ABORT:
+ td.stopDownload();
+ sleep();
+ break;
+ case RESTART:
+ td.stopDownload();
+ sleep();
+ threadPool.execute(td);
+ break;
+ case PURGE:
+ td.stopDownload();
+ answer =
+ new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
+ getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
+ jobs.remove(jobId);
+ return answer;
+ default:
+ break; // TODO
+ }
+ return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId),
+ getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
+ }
+
+ private String getInstallPath(String jobId) {
+ DownloadJob dj = jobs.get(jobId);
+ if (dj != null) {
+ return dj.getTmpltPath();
+ }
+ return null;
+ }
+
+ private List<String> listVolumes(String rootdir) {
+ List<String> result = new ArrayList<String>();
+
+ Script script = new Script(listVolScr, s_logger);
+ script.add("-r", rootdir);
+ ZfsPathParser zpp = new ZfsPathParser(rootdir);
+ script.execute(zpp);
+ result.addAll(zpp.getPaths());
+ s_logger.info("found " + zpp.getPaths().size() + " volumes" + zpp.getPaths());
+ return result;
+ }
+
+ private List<String> listTemplates(String rootdir) {
+ List<String> result = new ArrayList<String>();
+
+ Script script = new Script(listTmpltScr, s_logger);
+ script.add("-r", rootdir);
+ ZfsPathParser zpp = new ZfsPathParser(rootdir);
+ script.execute(zpp);
+ result.addAll(zpp.getPaths());
+ s_logger.info("found " + zpp.getPaths().size() + " templates" + zpp.getPaths());
+ return result;
+ }
+
+ @Override
+ public Map<String, TemplateProp> gatherTemplateInfo(String rootDir) {
+ Map<String, TemplateProp> result = new HashMap<String, TemplateProp>();
+ String templateDir = rootDir + File.separator + _templateDir;
+
+ if (!_storage.exists(templateDir)) {
+ _storage.mkdirs(templateDir);
+ }
+
+ List<String> publicTmplts = listTemplates(templateDir);
+ for (String tmplt : publicTmplts) {
+ String path = tmplt.substring(0, tmplt.lastIndexOf(File.separator));
+ TemplateLocation loc = new TemplateLocation(_storage, path);
+ try {
+ if (!loc.load()) {
+ s_logger.warn("Post download installation was not completed for " + path);
+ // loc.purge();
+ _storage.cleanup(path, templateDir);
+ continue;
+ }
+ } catch (IOException e) {
+ s_logger.warn("Unable to load template location " + path, e);
+ continue;
+ }
+
+ TemplateProp tInfo = loc.getTemplateInfo();
+
+ if ((tInfo.getSize() == tInfo.getPhysicalSize()) && (tInfo.getInstallPath().endsWith(ImageFormat.OVA.getFileExtension()))) {
+ try {
+ Processor processor = _processors.get("OVA Processor");
+ OVAProcessor vmdkProcessor = (OVAProcessor)processor;
+ long vSize = vmdkProcessor.getTemplateVirtualSize(path, tInfo.getInstallPath().substring(tInfo.getInstallPath().lastIndexOf(File.separator) + 1));
+ tInfo.setSize(vSize);
+ loc.updateVirtualSize(vSize);
+ loc.save();
+ } catch (Exception e) {
+ s_logger.error("Unable to get the virtual size of the template: " + tInfo.getInstallPath() + " due to " + e.getMessage());
+ }
+ }
+
+ result.put(tInfo.getTemplateName(), tInfo);
+ s_logger.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt);
+ }
+ /*
+ for (String tmplt : isoTmplts) {
+ String tmp[];
+ tmp = tmplt.split("/");
+ String tmpltName = tmp[tmp.length - 2];
+ tmplt = tmplt.substring(tmplt.lastIndexOf("iso/"));
+ TemplateInfo tInfo = new TemplateInfo(tmpltName, tmplt, false);
+ s_logger.debug("Added iso template name: " + tmpltName + ", path: " + tmplt);
+ result.put(tmpltName, tInfo);
+ }
+ */
+ return result;
+ }
+
+ @Override
+ public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {
+ Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>();
+ String volumeDir = rootDir + File.separator + _volumeDir;
+
+ if (!_storage.exists(volumeDir)) {
+ _storage.mkdirs(volumeDir);
+ }
+
+ List<String> vols = listVolumes(volumeDir);
+ for (String vol : vols) {
+ String path = vol.substring(0, vol.lastIndexOf(File.separator));
+ TemplateLocation loc = new TemplateLocation(_storage, path);
+ try {
+ if (!loc.load()) {
+ s_logger.warn("Post download installation was not completed for " + path);
+ // loc.purge();
+ _storage.cleanup(path, volumeDir);
+ continue;
+ }
+ } catch (IOException e) {
+ s_logger.warn("Unable to load volume location " + path, e);
+ continue;
+ }
+
+ TemplateProp vInfo = loc.getTemplateInfo();
+
+ if ((vInfo.getSize() == vInfo.getPhysicalSize()) && (vInfo.getInstallPath().endsWith(ImageFormat.OVA.getFileExtension()))) {
+ try {
+ Processor processor = _processors.get("OVA Processor");
+ OVAProcessor vmdkProcessor = (OVAProcessor)processor;
+ long vSize = vmdkProcessor.getTemplateVirtualSize(path, vInfo.getInstallPath().substring(vInfo.getInstallPath().lastIndexOf(File.separator) + 1));
+ vInfo.setSize(vSize);
+ loc.updateVirtualSize(vSize);
+ loc.save();
+ } catch (Exception e) {
+ s_logger.error("Unable to get the virtual size of the volume: " + vInfo.getInstallPath() + " due to " + e.getMessage());
+ }
+ }
+
+ result.put(vInfo.getId(), vInfo);
+ s_logger.debug("Added volume name: " + vInfo.getTemplateName() + ", path: " + vol);
+ }
+ return result;
+ }
+
+ public static class ZfsPathParser extends OutputInterpreter {
+ String _parent;
+ List<String> paths = new ArrayList<String>();
+
+ public ZfsPathParser(String parent) {
+ _parent = parent;
+ }
+
+ @Override
+ public String interpret(BufferedReader reader) throws IOException {
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ paths.add(line);
+ }
+ return null;
+ }
+
+ public List<String> getPaths() {
+ return paths;
+ }
+
+ @Override
+ public boolean drain() {
+ return true;
+ }
+ }
+
+ public DownloadManagerImpl() {
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+ _name = name;
+
+ String value = null;
+
+ _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
+ if (_storage == null) {
+ value = (String)params.get(StorageLayer.ClassConfigKey);
+ if (value == null) {
+ throw new ConfigurationException("Unable to find the storage layer");
+ }
+
+ Class<StorageLayer> clazz;
+ try {
+ clazz = (Class<StorageLayer>)Class.forName(value);
+ _storage = clazz.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new ConfigurationException("Unable to instantiate " + value);
+ } catch (InstantiationException e) {
+ throw new ConfigurationException("Unable to instantiate " + value);
+ } catch (IllegalAccessException e) {
+ throw new ConfigurationException("Unable to instantiate " + value);
+ }
+ }
+
+ String inSystemVM = (String)params.get("secondary.storage.vm");
+ if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
+ s_logger.info("DownloadManager: starting additional services since we are inside system vm");
+ startAdditionalServices();
+ blockOutgoingOnPrivate();
+ }
+
+ value = (String)params.get("install.timeout.pergig");
+ installTimeoutPerGig = NumbersUtil.parseInt(value, 15 * 60) * 1000;
+
+ value = (String)params.get("install.numthreads");
+ final int numInstallThreads = NumbersUtil.parseInt(value, 10);
+
+ String scriptsDir = (String)params.get("template.scripts.dir");
+ if (scriptsDir == null) {
+ scriptsDir = "scripts/storage/secondary";
+ }
+
+ listTmpltScr = Script.findScript(scriptsDir, "listvmtmplt.sh");
+ if (listTmpltScr == null) {
+ throw new ConfigurationException("Unable to find the listvmtmplt.sh");
+ }
+ s_logger.info("listvmtmplt.sh found in " + listTmpltScr);
+
+ createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
+ if (createTmpltScr == null) {
+ throw new ConfigurationException("Unable to find createtmplt.sh");
+ }
+ s_logger.info("createtmplt.sh found in " + createTmpltScr);
+
+ listVolScr = Script.findScript(scriptsDir, "listvolume.sh");
+ if (listVolScr == null) {
+ throw new ConfigurationException("Unable to find the listvolume.sh");
+ }
+ s_logger.info("listvolume.sh found in " + listVolScr);
+
+ createVolScr = Script.findScript(scriptsDir, "createvolume.sh");
+ if (createVolScr == null) {
+ throw new ConfigurationException("Unable to find createvolume.sh");
+ }
+ s_logger.info("createvolume.sh found in " + createVolScr);
+
+ _processors = new HashMap<String, Processor>();
+
+ Processor processor = new VhdProcessor();
+ processor.configure("VHD Processor", params);
+ _processors.put("VHD Processor", processor);
+
+ processor = new IsoProcessor();
+ processor.configure("ISO Processor", params);
+ _processors.put("ISO Processor", processor);
+
+ processor = new QCOW2Processor();
+ processor.configure("QCOW2 Processor", params);
+ _processors.put("QCOW2 Processor", processor);
+
+ processor = new OVAProcessor();
+ processor.configure("OVA Processor", params);
+ _processors.put("OVA Processor", processor);
+
+ processor = new VmdkProcessor();
+ processor.configure("VMDK Processor", params);
+ _processors.put("VMDK Processor", processor);
+
+ processor = new RawImageProcessor();
+ processor.configure("Raw Image Processor", params);
+ _processors.put("Raw Image Processor", processor);
+
+ _templateDir = (String)params.get("public.templates.root.dir");
+ if (_templateDir == null) {
+ _templateDir = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR;
+ }
+ _templateDir += File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR;
+ _volumeDir = TemplateConstants.DEFAULT_VOLUME_ROOT_DIR + File.separator;
+ // Add more processors here.
+ threadPool = Executors.newFixedThreadPool(numInstallThreads);
+ return true;
+ }
+
+ private void blockOutgoingOnPrivate() {
+ Script command = new Script("/bin/bash", s_logger);
+ String intf = "eth1";
+ command.add("-c");
+ command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" + "iptables -A OUTPUT -o " + intf +
+ " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;");
+
+ String result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result);
+ return;
+ }
+ }
+
+ @Override
+ public String getName() {
+ return _name;
+ }
+
+ @Override
+ public boolean start() {
+ return true;
+ }
+
+ @Override
+ public boolean stop() {
+ return true;
+ }
+
+ private void startAdditionalServices() {
+
+ Script command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("if [ -d /etc/apache2 ] ; then service apache2 stop; else service httpd stop; fi ");
+ String result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in stopping httpd service err=" + result);
+ }
+ String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
+ String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF;
+
+ command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" + "iptables -I INPUT -i " + intf +
+ " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;");
+
+ result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in opening up httpd port err=" + result);
+ return;
+ }
+
+ command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd start; fi ");
+ result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in starting httpd service err=" + result);
+ return;
+ }
+ command = new Script("mkdir", s_logger);
+ command.add("-p");
+ command.add("/var/www/html/copy/template");
+ result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in creating directory =" + result);
+ return;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManager.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManager.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManager.java
new file mode 100755
index 0000000..be99fea
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManager.java
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+
+import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
+import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer;
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.Upload.Status;
+import com.cloud.storage.template.TemplateUploader;
+import com.cloud.utils.component.Manager;
+
+public interface UploadManager extends Manager {
+
+ /**
+ * @param jobId job Id
+ * @return status of the upload job
+ */
+ public TemplateUploader.Status getUploadStatus(String jobId);
+
+ /**
+ * @param jobId job Id
+ * @return status of the upload job
+ */
+ public Status getUploadStatus2(String jobId);
+
+ /**
+ * Get the upload percent of a upload job
+ * @param jobId job Id
+ * @return
+ */
+ public int getUploadPct(String jobId);
+
+ /**
+ * Get the upload error if any
+ * @param jobId job Id
+ * @return
+ */
+ public String getUploadError(String jobId);
+
+ /**
+ * Get the local path for the upload
+ * @param jobId job Id
+ * @return
+ public String getUploadLocalPath(String jobId);
+ */
+
+ /** Handle upload commands from the management server
+ * @param cmd cmd from server
+ * @return answer representing status of upload.
+ */
+ public UploadAnswer handleUploadCommand(SecondaryStorageResource resource, UploadCommand cmd);
+
+ public String getPublicTemplateRepo();
+
+ String uploadPublicTemplate(long id, String url, String name, ImageFormat format, Long accountId, String descr, String cksum, String installPathPrefix, String user,
+ String password, long maxTemplateSizeInBytes);
+
+ CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd);
+
+ DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd);
+
+}