You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ah...@apache.org on 2014/02/05 02:39:42 UTC
[04/11] Moved the secondary storage service into its own server
directory
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java
new file mode 100755
index 0000000..cdbc52d
--- /dev/null
+++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadManagerImpl.java
@@ -0,0 +1,550 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import org.apache.cloudstack.storage.resource.SecondaryStorageResource;
+
+import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
+import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer;
+import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.agent.api.storage.UploadProgressCommand;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.StorageLayer;
+import com.cloud.storage.Upload;
+import com.cloud.storage.UploadVO;
+import com.cloud.storage.template.FtpTemplateUploader;
+import com.cloud.storage.template.TemplateUploader;
+import com.cloud.storage.template.TemplateUploader.Status;
+import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback;
+import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+
+public class UploadManagerImpl extends ManagerBase implements UploadManager {
+
+ public class Completion implements UploadCompleteCallback {
+ private final String jobId;
+
+ public Completion(String jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public void uploadComplete(Status status) {
+ setUploadStatus(jobId, status);
+ }
+ }
+
+ private static class UploadJob {
+ private final TemplateUploader tu;
+
+ public UploadJob(TemplateUploader tu, String jobId, long id, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum,
+ String installPathPrefix) {
+ super();
+ this.tu = tu;
+ }
+
+ public TemplateUploader getTemplateUploader() {
+ return tu;
+ }
+
+ public void cleanup() {
+ if (tu != null) {
+ String upldPath = tu.getUploadLocalPath();
+ if (upldPath != null) {
+ File f = new File(upldPath);
+ f.delete();
+ }
+ }
+ }
+
+ }
+
+ public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class);
+ private ExecutorService threadPool;
+ private final Map<String, UploadJob> jobs = new ConcurrentHashMap<String, UploadJob>();
+ private String parentDir;
+ private final String extractMountPoint = "/mnt/SecStorage/extractmnt";
+ private StorageLayer _storage;
+ private boolean hvm;
+
+ @Override
+ public String uploadPublicTemplate(long id, String url, String name, ImageFormat format, Long accountId, String descr, String cksum, String installPathPrefix,
+ String userName, String passwd, long templateSizeInBytes) {
+
+ UUID uuid = UUID.randomUUID();
+ String jobId = uuid.toString();
+
+ String completePath = parentDir + File.separator + installPathPrefix;
+ s_logger.debug("Starting upload from " + completePath);
+
+ URI uri;
+ try {
+ uri = new URI(url);
+ } catch (URISyntaxException e) {
+ s_logger.error("URI is incorrect: " + url);
+ throw new CloudRuntimeException("URI is incorrect: " + url);
+ }
+ TemplateUploader tu;
+ if ((uri != null) && (uri.getScheme() != null)) {
+ if (uri.getScheme().equalsIgnoreCase("ftp")) {
+ tu = new FtpTemplateUploader(completePath, url, new Completion(jobId), templateSizeInBytes);
+ } else {
+ s_logger.error("Scheme is not supported " + url);
+ throw new CloudRuntimeException("Scheme is not supported " + url);
+ }
+ } else {
+ s_logger.error("Unable to download from URL: " + url);
+ throw new CloudRuntimeException("Unable to download from URL: " + url);
+ }
+ UploadJob uj = new UploadJob(tu, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix);
+ jobs.put(jobId, uj);
+ threadPool.execute(tu);
+
+ return jobId;
+
+ }
+
+ @Override
+ public String getUploadError(String jobId) {
+ UploadJob uj = jobs.get(jobId);
+ if (uj != null) {
+ return uj.getTemplateUploader().getUploadError();
+ }
+ return null;
+ }
+
+ @Override
+ public int getUploadPct(String jobId) {
+ UploadJob uj = jobs.get(jobId);
+ if (uj != null) {
+ return uj.getTemplateUploader().getUploadPercent();
+ }
+ return 0;
+ }
+
+ @Override
+ public Status getUploadStatus(String jobId) {
+ UploadJob job = jobs.get(jobId);
+ if (job != null) {
+ TemplateUploader tu = job.getTemplateUploader();
+ if (tu != null) {
+ return tu.getStatus();
+ }
+ }
+ return Status.UNKNOWN;
+ }
+
+ public static UploadVO.Status convertStatus(Status tds) {
+ switch (tds) {
+ case ABORTED:
+ return UploadVO.Status.NOT_UPLOADED;
+ case UPLOAD_FINISHED:
+ return UploadVO.Status.UPLOAD_IN_PROGRESS;
+ case IN_PROGRESS:
+ return UploadVO.Status.UPLOAD_IN_PROGRESS;
+ case NOT_STARTED:
+ return UploadVO.Status.NOT_UPLOADED;
+ case RECOVERABLE_ERROR:
+ return UploadVO.Status.NOT_UPLOADED;
+ case UNKNOWN:
+ return UploadVO.Status.UNKNOWN;
+ case UNRECOVERABLE_ERROR:
+ return UploadVO.Status.UPLOAD_ERROR;
+ case POST_UPLOAD_FINISHED:
+ return UploadVO.Status.UPLOADED;
+ default:
+ return UploadVO.Status.UNKNOWN;
+ }
+ }
+
+ @Override
+ public com.cloud.storage.UploadVO.Status getUploadStatus2(String jobId) {
+ return convertStatus(getUploadStatus(jobId));
+ }
+
+ @Override
+ public String getPublicTemplateRepo() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ private UploadAnswer handleUploadProgressCmd(UploadProgressCommand cmd) {
+ String jobId = cmd.getJobId();
+ UploadAnswer answer;
+ UploadJob uj = null;
+ if (jobId != null)
+ uj = jobs.get(jobId);
+ if (uj == null) {
+ return new UploadAnswer(null, 0, "Cannot find job", com.cloud.storage.UploadVO.Status.UNKNOWN, "", "", 0);
+ }
+ TemplateUploader td = uj.getTemplateUploader();
+ switch (cmd.getRequest()) {
+ case GET_STATUS:
+ break;
+ case ABORT:
+ td.stopUpload();
+ sleep();
+ break;
+ /*case RESTART:
+ td.stopUpload();
+ sleep();
+ threadPool.execute(td);
+ break;*/
+ case PURGE:
+ td.stopUpload();
+ answer =
+ new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+ getUploadTemplateSize(jobId));
+ jobs.remove(jobId);
+ return answer;
+ default:
+ break; // TODO
+ }
+ return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+ getUploadTemplateSize(jobId));
+ }
+
+ @Override
+ public UploadAnswer handleUploadCommand(SecondaryStorageResource resource, UploadCommand cmd) {
+ s_logger.warn("Handling the upload " + cmd.getInstallPath() + " " + cmd.getId());
+ if (cmd instanceof UploadProgressCommand) {
+ return handleUploadProgressCmd((UploadProgressCommand)cmd);
+ }
+
+ String user = null;
+ String password = null;
+ String jobId =
+ uploadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(),
+ cmd.getInstallPath(), user, password, cmd.getTemplateSizeInBytes());
+ sleep();
+ return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+ getUploadTemplateSize(jobId));
+ }
+
+ @Override
+ public CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd) {
+
+ boolean isApacheUp = checkAndStartApache();
+ if (!isApacheUp) {
+ String errorString = "Error in starting Apache server ";
+ s_logger.error(errorString);
+ return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+ }
+ // Create the directory structure so that its visible under apache server root
+ String extractDir = "/var/www/html/userdata/";
+ Script command = new Script("mkdir", s_logger);
+ command.add("-p");
+ command.add(extractDir);
+ String result = command.execute();
+ if (result != null) {
+ String errorString = "Error in creating directory =" + result;
+ s_logger.error(errorString);
+ return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+ }
+
+ // Create a random file under the directory for security reasons.
+ String uuid = cmd.getExtractLinkUUID();
+ command = new Script("touch", s_logger);
+ command.add(extractDir + uuid);
+ result = command.execute();
+ if (result != null) {
+ String errorString = "Error in creating file " + uuid + " ,error: " + result;
+ s_logger.warn(errorString);
+ return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+ }
+
+ // Create a symbolic link from the actual directory to the template location. The entity would be directly visible under /var/www/html/userdata/cmd.getInstallPath();
+ command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("ln -sf /mnt/SecStorage/" + cmd.getParent() + File.separator + cmd.getInstallPath() + " " + extractDir + uuid);
+ result = command.execute();
+ if (result != null) {
+ String errorString = "Error in linking err=" + result;
+ s_logger.error(errorString);
+ return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+ }
+
+ return new CreateEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS);
+
+ }
+
+ @Override
+ public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd) {
+
+ //Delete the soft link. Example path = volumes/8/74eeb2c6-8ab1-4357-841f-2e9d06d1f360.vhd
+ s_logger.warn("handleDeleteEntityDownloadURLCommand Path:" + cmd.getPath() + " Type:" + cmd.getType().toString());
+ String path = cmd.getPath();
+ Script command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+
+ //We just need to remove the UUID.vhd
+ String extractUrl = cmd.getExtractUrl();
+ command.add("unlink /var/www/html/userdata/" + extractUrl.substring(extractUrl.lastIndexOf(File.separator) + 1));
+ String result = command.execute();
+ if (result != null) {
+ String errorString = "Error in deleting =" + result;
+ s_logger.warn(errorString);
+ return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+ }
+
+ // If its a volume also delete the Hard link since it was created only for the purpose of download.
+ if (cmd.getType() == Upload.Type.VOLUME) {
+ command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("rm -f /mnt/SecStorage/" + cmd.getParentPath() + File.separator + path);
+ s_logger.warn(" " + parentDir + File.separator + path);
+ result = command.execute();
+ if (result != null) {
+ String errorString = "Error in linking err=" + result;
+ s_logger.warn(errorString);
+ return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
+ }
+ }
+
+ return new DeleteEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS);
+ }
+
+ private String getInstallPath(String jobId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ private String getUploadLocalPath(String jobId) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ private long getUploadTemplateSize(String jobId) {
+ return 0;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
+
+ String value = null;
+
+ _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
+ if (_storage == null) {
+ value = (String)params.get(StorageLayer.ClassConfigKey);
+ if (value == null) {
+ throw new ConfigurationException("Unable to find the storage layer");
+ }
+
+ Class<StorageLayer> clazz;
+ try {
+ clazz = (Class<StorageLayer>)Class.forName(value);
+ _storage = clazz.newInstance();
+ } catch (ClassNotFoundException e) {
+ throw new ConfigurationException("Unable to instantiate " + value);
+ } catch (InstantiationException e) {
+ throw new ConfigurationException("Unable to instantiate " + value);
+ } catch (IllegalAccessException e) {
+ throw new ConfigurationException("Unable to instantiate " + value);
+ }
+ }
+
+ String inSystemVM = (String)params.get("secondary.storage.vm");
+ if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
+ s_logger.info("UploadManager: starting additional services since we are inside system vm");
+ startAdditionalServices();
+ //blockOutgoingOnPrivate();
+ }
+
+ value = (String)params.get("install.numthreads");
+ final int numInstallThreads = NumbersUtil.parseInt(value, 10);
+
+ String scriptsDir = (String)params.get("template.scripts.dir");
+ if (scriptsDir == null) {
+ scriptsDir = "scripts/storage/secondary";
+ }
+
+ // Add more processors here.
+ threadPool = Executors.newFixedThreadPool(numInstallThreads);
+
+ return true;
+ }
+
+ private void startAdditionalServices() {
+
+ Script command = new Script("rm", s_logger);
+ command.add("-rf");
+ command.add(extractMountPoint);
+ String result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in creating file " + extractMountPoint + " ,error: " + result);
+ return;
+ }
+
+ command = new Script("touch", s_logger);
+ command.add(extractMountPoint);
+ result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in creating file " + extractMountPoint + " ,error: " + result);
+ return;
+ }
+
+ command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("ln -sf " + parentDir + " " + extractMountPoint);
+ result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in linking err=" + result);
+ return;
+ }
+
+ }
+
+ /**
+ * Get notified of change of job status. Executed in context of uploader thread
+ *
+ * @param jobId
+ * the id of the job
+ * @param status
+ * the status of the job
+ */
+ public void setUploadStatus(String jobId, Status status) {
+ UploadJob uj = jobs.get(jobId);
+ if (uj == null) {
+ s_logger.warn("setUploadStatus for jobId: " + jobId + ", status=" + status + " no job found");
+ return;
+ }
+ TemplateUploader tu = uj.getTemplateUploader();
+ s_logger.warn("Upload Completion for jobId: " + jobId + ", status=" + status);
+ s_logger.warn("UploadedBytes=" + tu.getUploadedBytes() + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent());
+
+ switch (status) {
+ case ABORTED:
+ case NOT_STARTED:
+ case UNRECOVERABLE_ERROR:
+ // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume.
+ if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) {
+ uj.cleanup();
+ }
+ break;
+ case UNKNOWN:
+ return;
+ case IN_PROGRESS:
+ s_logger.info("Resuming jobId: " + jobId + ", status=" + status);
+ tu.setResume(true);
+ threadPool.execute(tu);
+ break;
+ case RECOVERABLE_ERROR:
+ threadPool.execute(tu);
+ break;
+ case UPLOAD_FINISHED:
+ tu.setUploadError("Upload success, starting install ");
+ String result = postUpload(jobId);
+ if (result != null) {
+ s_logger.error("Failed post upload script: " + result);
+ tu.setStatus(Status.UNRECOVERABLE_ERROR);
+ tu.setUploadError("Failed post upload script: " + result);
+ } else {
+ s_logger.warn("Upload completed successfully at " + new SimpleDateFormat().format(new Date()));
+ tu.setStatus(Status.POST_UPLOAD_FINISHED);
+ tu.setUploadError("Upload completed successfully at " + new SimpleDateFormat().format(new Date()));
+ }
+ // Delete the entity only if its a volume. TO DO - find a better way of finding it a volume.
+ if (uj.getTemplateUploader().getUploadLocalPath().indexOf("volume") > -1) {
+ uj.cleanup();
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ private String postUpload(String jobId) {
+ return null;
+ }
+
+ private void sleep() {
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+
+ private boolean checkAndStartApache() {
+
+ //Check whether the Apache server is running
+ Script command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("if [ -d /etc/apache2 ] ; then service apache2 status | grep pid; else service httpd status | grep pid; fi ");
+ String result = command.execute();
+
+ //Apache Server is not running. Try to start it.
+ if (result != null) {
+
+ /*s_logger.warn("Apache server not running, trying to start it");
+ String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
+ String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF;
+
+ command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j DROP;" +
+ "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j HTTP;" +
+ "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j DROP;" +
+ "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j HTTP;" +
+ "iptables -F HTTP;" +
+ "iptables -X HTTP;" +
+ "iptables -N HTTP;" +
+ "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j DROP;" +
+ "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j DROP;" +
+ "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j HTTP;" +
+ "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j HTTP;");
+
+ result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in opening up httpd port err=" + result );
+ return false;
+ }*/
+
+ command = new Script("/bin/bash", s_logger);
+ command.add("-c");
+ command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd start; fi ");
+ result = command.execute();
+ if (result != null) {
+ s_logger.warn("Error in starting httpd service err=" + result);
+ return false;
+ }
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
new file mode 100644
index 0000000..e0fcbae
--- /dev/null
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.resource;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import javax.naming.ConfigurationException;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.log4j.Logger;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.cloudstack.storage.command.CopyCmdAnswer;
+import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.ListTemplateAnswer;
+import com.cloud.agent.api.storage.ListTemplateCommand;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class LocalNfsSecondaryStorageResourceTest extends TestCase {
+ private static Map<String, Object> testParams;
+
+ private static final Logger s_logger = Logger.getLogger(LocalNfsSecondaryStorageResourceTest.class.getName());
+
+ LocalNfsSecondaryStorageResource resource;
+
+ @Before
+ @Override
+ public void setUp() throws ConfigurationException {
+ resource = new LocalNfsSecondaryStorageResource();
+ resource.setInSystemVM(true);
+
+ testParams = PropertiesUtil.toMap(loadProperties());
+ resource.configureStorageLayerClass(testParams);
+ Object testLocalRoot = testParams.get("testLocalRoot");
+ resource.setParentPath("/mnt");
+
+ if (testLocalRoot != null) {
+ resource.setParentPath((String)testLocalRoot);
+ }
+
+ System.setProperty("paths.script", "/Users/edison/develop/asf-master/script");
+ //resource.configure("test", new HashMap<String, Object>());
+ }
+
+ @Test
+ public void testExecuteRequest() throws Exception {
+ TemplateObjectTO template = Mockito.mock(TemplateObjectTO.class);
+ NfsTO cacheStore = Mockito.mock(NfsTO.class);
+ Mockito.when(cacheStore.getUrl()).thenReturn("nfs://nfs2.lab.vmops.com/export/home/edison/");
+ SwiftTO swift = Mockito.mock(SwiftTO.class);
+ Mockito.when(swift.getEndPoint()).thenReturn("https://objects.dreamhost.com/auth");
+ Mockito.when(swift.getAccount()).thenReturn("cloudstack");
+ Mockito.when(swift.getUserName()).thenReturn("images");
+ Mockito.when(swift.getKey()).thenReturn("oxvELQaOD1U5_VyosGfA-wpZ7uBWEff-CUBGCM0u");
+
+ Mockito.when(template.getDataStore()).thenReturn(swift);
+ Mockito.when(template.getPath()).thenReturn("template/1/1/");
+ Mockito.when(template.isRequiresHvm()).thenReturn(true);
+ Mockito.when(template.getId()).thenReturn(1L);
+ Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.VHD);
+ Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/test.bz2");
+ Mockito.when(template.getName()).thenReturn(UUID.randomUUID().toString());
+ Mockito.when(template.getObjectType()).thenReturn(DataObjectType.TEMPLATE);
+
+ DownloadCommand cmd = new DownloadCommand(template, 100000L);
+ cmd.setCacheStore(cacheStore);
+ DownloadAnswer answer = (DownloadAnswer)resource.executeRequest(cmd);
+ Assert.assertTrue(answer.getResult());
+
+ Mockito.when(template.getPath()).thenReturn(answer.getInstallPath());
+ Mockito.when(template.getDataStore()).thenReturn(swift);
+ //download swift:
+ Mockito.when(cacheStore.getRole()).thenReturn(DataStoreRole.ImageCache);
+ TemplateObjectTO destTemplate = Mockito.mock(TemplateObjectTO.class);
+ Mockito.when(destTemplate.getPath()).thenReturn("template/1/2");
+ Mockito.when(destTemplate.getDataStore()).thenReturn(cacheStore);
+ Mockito.when(destTemplate.getObjectType()).thenReturn(DataObjectType.TEMPLATE);
+ CopyCommand cpyCmd = new CopyCommand(template, destTemplate, 10000, true);
+ CopyCmdAnswer copyCmdAnswer = (CopyCmdAnswer)resource.executeRequest(cpyCmd);
+ Assert.assertTrue(copyCmdAnswer.getResult());
+
+ //list template
+ ListTemplateCommand listCmd = new ListTemplateCommand(swift);
+ ListTemplateAnswer listAnswer = (ListTemplateAnswer)resource.executeRequest(listCmd);
+
+ Assert.assertTrue(listAnswer.getTemplateInfo().size() > 0);
+ }
+
+ public static Properties loadProperties() throws ConfigurationException {
+ Properties properties = new Properties();
+ final File file = PropertiesUtil.findConfigFile("agent.properties");
+ if (file == null) {
+ throw new ConfigurationException("Unable to find agent.properties.");
+ }
+
+ s_logger.info("agent.properties found at " + file.getAbsolutePath());
+
+ try {
+ properties.load(new FileInputStream(file));
+ } catch (final FileNotFoundException ex) {
+ throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex);
+ } catch (final IOException ex) {
+ throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex);
+ }
+ return properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
new file mode 100644
index 0000000..e0ae4c5
--- /dev/null
+++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.resource;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.naming.ConfigurationException;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class NfsSecondaryStorageResourceTest extends TestCase {
+ private static Map<String, Object> testParams;
+
+ private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName());
+
+ NfsSecondaryStorageResource resource;
+
+ @Before
+ @Override
+ public void setUp() throws ConfigurationException {
+ s_logger.setLevel(Level.ALL);
+ resource = new NfsSecondaryStorageResource();
+ resource.setInSystemVM(true);
+ testParams = PropertiesUtil.toMap(loadProperties());
+ resource.configureStorageLayerClass(testParams);
+ Object testLocalRoot = testParams.get("testLocalRoot");
+ if (testLocalRoot != null) {
+ resource.setParentPath((String)testLocalRoot);
+ }
+ }
+
+ @Test
+ public void testMount() throws Exception {
+ String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar";
+ URI sampleUri = new URI(sampleUriStr);
+
+ s_logger.info("Check HostIp parsing");
+ String hostIpStr = resource.getUriHostIp(sampleUri);
+ Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr);
+
+ s_logger.info("Check option parsing");
+ String expected = "user=administrator,password=1pass@word1,foo=bar,";
+ String actualOpts = resource.parseCifsMountOptions(sampleUri);
+ Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts);
+
+ // attempt a configured mount
+ final Map<String, Object> params = PropertiesUtil.toMap(loadProperties());
+ String sampleMount = (String)params.get("testCifsMount");
+ if (!sampleMount.isEmpty()) {
+ s_logger.info("functional test, mount " + sampleMount);
+ URI realMntUri = new URI(sampleMount);
+ String mntSubDir = resource.mountUri(realMntUri);
+ s_logger.info("functional test, umount " + mntSubDir);
+ resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri);
+ } else {
+ s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test");
+ }
+ }
+
+ public static Properties loadProperties() throws ConfigurationException {
+ Properties properties = new Properties();
+ final File file = PropertiesUtil.findConfigFile("agent.properties");
+ if (file == null) {
+ throw new ConfigurationException("Unable to find agent.properties.");
+ }
+
+ s_logger.info("agent.properties found at " + file.getAbsolutePath());
+
+ try {
+ properties.load(new FileInputStream(file));
+ } catch (final FileNotFoundException ex) {
+ throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex);
+ } catch (final IOException ex) {
+ throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex);
+ }
+ return properties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java
deleted file mode 100644
index 9393ee2..0000000
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java
+++ /dev/null
@@ -1,95 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.apache.cloudstack.storage.resource;
-
-import java.net.URI;
-import java.util.concurrent.Executors;
-
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.Command;
-import com.cloud.storage.JavaStorageLayer;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.script.Script;
-
-@Component
-public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResource {
-
- private static final Logger s_logger = Logger.getLogger(LocalNfsSecondaryStorageResource.class);
-
- public LocalNfsSecondaryStorageResource() {
- this._dlMgr = new DownloadManagerImpl();
- ((DownloadManagerImpl)_dlMgr).setThreadPool(Executors.newFixedThreadPool(10));
- _storage = new JavaStorageLayer();
- this._inSystemVM = false;
- }
-
- @Override
- public void setParentPath(String path) {
- this._parent = path;
- }
-
- @Override
- public Answer executeRequest(Command cmd) {
- return super.executeRequest(cmd);
- }
-
- @Override
- synchronized public String getRootDir(String secUrl) {
- try {
- URI uri = new URI(secUrl);
- String dir = mountUri(uri);
- return _parent + "/" + dir;
- } catch (Exception e) {
- String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString();
- s_logger.error(msg, e);
- throw new CloudRuntimeException(msg);
- }
- }
-
- @Override
- protected void mount(String localRootPath, String remoteDevice, URI uri) {
- ensureLocalRootPathExists(localRootPath, uri);
-
- if (mountExists(localRootPath, uri)) {
- return;
- }
-
- attemptMount(localRootPath, remoteDevice, uri);
-
- // Change permissions for the mountpoint - seems to bypass authentication
- Script script = new Script(true, "chmod", _timeout, s_logger);
- script.add("777", localRootPath);
- String result = script.execute();
- if (result != null) {
- String errMsg = "Unable to set permissions for " + localRootPath + " due to " + result;
- s_logger.error(errMsg);
- throw new CloudRuntimeException(errMsg);
- }
- s_logger.debug("Successfully set 777 permission for " + localRootPath);
-
- // XXX: Adding the check for creation of snapshots dir here. Might have
- // to move it somewhere more logical later.
- checkForSnapshotsDir(localRootPath);
- checkForVolumesDir(localRootPath);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/4be3b993/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java
----------------------------------------------------------------------
diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java
deleted file mode 100644
index bdfe7e8..0000000
--- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalSecondaryStorageResource.java
+++ /dev/null
@@ -1,240 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package org.apache.cloudstack.storage.resource;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.naming.ConfigurationException;
-
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.storage.command.DownloadCommand;
-import org.apache.cloudstack.storage.command.DownloadProgressCommand;
-import org.apache.cloudstack.storage.template.DownloadManager;
-import org.apache.cloudstack.storage.template.DownloadManagerImpl;
-
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.CheckHealthAnswer;
-import com.cloud.agent.api.CheckHealthCommand;
-import com.cloud.agent.api.Command;
-import com.cloud.agent.api.ComputeChecksumCommand;
-import com.cloud.agent.api.PingCommand;
-import com.cloud.agent.api.PingStorageCommand;
-import com.cloud.agent.api.ReadyAnswer;
-import com.cloud.agent.api.ReadyCommand;
-import com.cloud.agent.api.SecStorageSetupCommand;
-import com.cloud.agent.api.StartupCommand;
-import com.cloud.agent.api.StartupStorageCommand;
-import com.cloud.agent.api.storage.ListTemplateAnswer;
-import com.cloud.agent.api.storage.ListTemplateCommand;
-import com.cloud.agent.api.to.NfsTO;
-import com.cloud.host.Host;
-import com.cloud.host.Host.Type;
-import com.cloud.resource.ServerResourceBase;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.StoragePoolType;
-import com.cloud.storage.StorageLayer;
-import com.cloud.storage.template.TemplateProp;
-import com.cloud.utils.component.ComponentContext;
-
-public class LocalSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
- private static final Logger s_logger = Logger.getLogger(LocalSecondaryStorageResource.class);
- int _timeout;
-
- String _instance;
- String _parent;
-
- String _dc;
- String _pod;
- String _guid;
-
- StorageLayer _storage;
-
- DownloadManager _dlMgr;
-
- @Override
- public void disconnected() {
- }
-
- @Override
- public String getRootDir(String url) {
- return getRootDir();
-
- }
-
- public String getRootDir() {
- return _parent;
- }
-
- @Override
- public Answer executeRequest(Command cmd) {
- if (cmd instanceof DownloadProgressCommand) {
- return _dlMgr.handleDownloadCommand(this, (DownloadProgressCommand)cmd);
- } else if (cmd instanceof DownloadCommand) {
- return _dlMgr.handleDownloadCommand(this, (DownloadCommand)cmd);
- } else if (cmd instanceof CheckHealthCommand) {
- return new CheckHealthAnswer((CheckHealthCommand)cmd, true);
- } else if (cmd instanceof SecStorageSetupCommand) {
- return new Answer(cmd, true, "success");
- } else if (cmd instanceof ReadyCommand) {
- return new ReadyAnswer((ReadyCommand)cmd);
- } else if (cmd instanceof ListTemplateCommand) {
- return execute((ListTemplateCommand)cmd);
- } else if (cmd instanceof ComputeChecksumCommand) {
- return execute((ComputeChecksumCommand)cmd);
- } else {
- return Answer.createUnsupportedCommandAnswer(cmd);
- }
- }
-
- private Answer execute(ComputeChecksumCommand cmd) {
- return new Answer(cmd, false, null);
- }
-
- private Answer execute(ListTemplateCommand cmd) {
- String root = getRootDir();
- Map<String, TemplateProp> templateInfos = _dlMgr.gatherTemplateInfo(root);
- return new ListTemplateAnswer(((NfsTO)cmd.getDataStore()).getUrl(), templateInfos);
- }
-
- @Override
- public Type getType() {
- return Host.Type.LocalSecondaryStorage;
- }
-
- @Override
- public PingCommand getCurrentStatus(final long id) {
- return new PingStorageCommand(Host.Type.Storage, id, new HashMap<String, Boolean>());
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
- super.configure(name, params);
-
- _guid = (String)params.get("guid");
- if (_guid == null) {
- throw new ConfigurationException("Unable to find the guid");
- }
-
- _dc = (String)params.get("zone");
- if (_dc == null) {
- throw new ConfigurationException("Unable to find the zone");
- }
- _pod = (String)params.get("pod");
-
- _instance = (String)params.get("instance");
-
- _parent = (String)params.get("mount.path");
- if (_parent == null) {
- throw new ConfigurationException("No directory specified.");
- }
-
- _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
- if (_storage == null) {
- String value = (String)params.get(StorageLayer.ClassConfigKey);
- if (value == null) {
- value = "com.cloud.storage.JavaStorageLayer";
- }
-
- try {
- Class<StorageLayer> clazz = (Class<StorageLayer>)Class.forName(value);
- _storage = ComponentContext.inject(clazz);
- } catch (ClassNotFoundException e) {
- throw new ConfigurationException("Unable to find class " + value);
- }
- }
-
- if (!_storage.mkdirs(_parent)) {
- s_logger.warn("Unable to create the directory " + _parent);
- throw new ConfigurationException("Unable to create the directory " + _parent);
- }
-
- s_logger.info("Mount point established at " + _parent);
-
- params.put("template.parent", _parent);
- params.put(StorageLayer.InstanceConfigKey, _storage);
-
- _dlMgr = new DownloadManagerImpl();
- _dlMgr.configure("DownloadManager", params);
-
- return true;
- }
-
- @Override
- public boolean start() {
- return true;
- }
-
- @Override
- public boolean stop() {
- return true;
- }
-
- @Override
- public StartupCommand[] initialize() {
-
- final StartupStorageCommand cmd =
- new StartupStorageCommand(_parent, StoragePoolType.Filesystem, 1024l * 1024l * 1024l * 1024l, _dlMgr.gatherTemplateInfo(_parent));
- cmd.setResourceType(Storage.StorageResourceType.LOCAL_SECONDARY_STORAGE);
- cmd.setIqn("local://");
- fillNetworkInformation(cmd);
- cmd.setDataCenter(_dc);
- cmd.setPod(_pod);
- cmd.setGuid(_guid);
- cmd.setName(_guid);
- cmd.setVersion(LocalSecondaryStorageResource.class.getPackage().getImplementationVersion());
-
- return new StartupCommand[] {cmd};
- }
-
- @Override
- protected String getDefaultScriptsDir() {
- return "scripts/storage/secondary";
- }
-
- @Override
- public void setName(String name) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void setConfigParams(Map<String, Object> params) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public Map<String, Object> getConfigParams() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override
- public int getRunLevel() {
- // TODO Auto-generated method stub
- return 0;
- }
-
- @Override
- public void setRunLevel(int level) {
- // TODO Auto-generated method stub
-
- }
-}