You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ed...@apache.org on 2012/12/20 03:43:29 UTC
git commit: trying to add code to download url into devcloud
Updated Branches:
refs/heads/javelin 8aaf5ba3a -> c216990e1
trying to add code to download url into devcloud
Project: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/commit/c216990e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/tree/c216990e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/diff/c216990e
Branch: refs/heads/javelin
Commit: c216990e1c2ca1eda40f367ed774f27b31980be4
Parents: 8aaf5ba
Author: Edison Su <su...@gmail.com>
Authored: Wed Dec 19 18:43:04 2012 -0800
Committer: Edison Su <su...@gmail.com>
Committed: Wed Dec 19 18:43:04 2012 -0800
----------------------------------------------------------------------
engine/storage/integration-test/pom.xml | 6 +
.../cloudstack/storage/test/DirectAgentTest.java | 33 ++++
.../apache/cloudstack/storage/test/TestHttp.java | 88 +++++++++++
.../apache/cloudstack/storage/to/TemplateTO.java | 7 +-
.../xen/resource/XenServerStorageResource.java | 116 +++++++++++----
scripts/vm/hypervisor/xenserver/storagePlugin | 56 +++++++
6 files changed, 273 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/c216990e/engine/storage/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/engine/storage/integration-test/pom.xml b/engine/storage/integration-test/pom.xml
index 947614c..0dbc9a2 100644
--- a/engine/storage/integration-test/pom.xml
+++ b/engine/storage/integration-test/pom.xml
@@ -51,6 +51,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.2.2</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${cs.mysql.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/c216990e/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java
----------------------------------------------------------------------
diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java
index 852f379..2f5e7ae 100644
--- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java
+++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/DirectAgentTest.java
@@ -22,9 +22,15 @@ import java.util.UUID;
import javax.inject.Inject;
+import org.apache.cloudstack.storage.command.CopyTemplateToPrimaryStorageCmd;
+import org.apache.cloudstack.storage.to.ImageDataStoreTO;
+import org.apache.cloudstack.storage.to.ImageOnPrimayDataStoreTO;
+import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
+import org.apache.cloudstack.storage.to.TemplateTO;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.Mockito;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@@ -122,4 +128,31 @@ public class DirectAgentTest {
e.printStackTrace();
}
}
+
+ @Test
+ public void testDownloadTemplate() {
+ ImageOnPrimayDataStoreTO image = Mockito.mock(ImageOnPrimayDataStoreTO.class);
+ PrimaryDataStoreTO primaryStore = Mockito.mock(PrimaryDataStoreTO.class);
+ Mockito.when(primaryStore.getUuid()).thenReturn("cd10cac1-4772-92e5-5da6-c2bc16b1ce1b");
+ Mockito.when(image.getPrimaryDataStore()).thenReturn(primaryStore);
+
+ ImageDataStoreTO imageStore = Mockito.mock(ImageDataStoreTO.class);
+ Mockito.when(imageStore.getType()).thenReturn("http");
+
+ TemplateTO template = Mockito.mock(TemplateTO.class);
+ Mockito.when(template.getPath()).thenReturn("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd");
+ Mockito.when(template.getImageDataStore()).thenReturn(imageStore);
+
+ Mockito.when(image.getTemplate()).thenReturn(template);
+ CopyTemplateToPrimaryStorageCmd cmd = new CopyTemplateToPrimaryStorageCmd(image);
+ try {
+ agentMgr.send(hostId, cmd);
+ } catch (AgentUnavailableException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (OperationTimedoutException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/c216990e/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java
----------------------------------------------------------------------
diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java
new file mode 100644
index 0000000..9de1c21
--- /dev/null
+++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/TestHttp.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cloudstack.storage.test;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.FileChannel;
+
+import junit.framework.Assert;
+
+import org.apache.commons.httpclient.HttpException;
+import org.apache.cxf.helpers.IOUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations="classpath:/resource/storageContext.xml")
+public class TestHttp {
+ @Test
+ public void testHttpclient() {
+ HttpHead method = new HttpHead("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd");
+ DefaultHttpClient client = new DefaultHttpClient();
+
+ OutputStream output = null;
+ long length = 0;
+ try {
+ HttpResponse response = client.execute(method);
+ length = Long.parseLong(response.getFirstHeader("Content-Length").getValue());
+ System.out.println(response.getFirstHeader("Content-Length").getValue());
+ File localFile = new File("/tmp/test");
+ if (!localFile.exists()) {
+ localFile.createNewFile();
+ }
+
+ HttpGet getMethod = new HttpGet("http://download.cloud.com/templates/devcloud/defaulttemplates/5/ce5b212e-215a-3461-94fb-814a635b2215.vhd");
+ response = client.execute(getMethod);
+ HttpEntity entity = response.getEntity();
+
+ output = new BufferedOutputStream(new FileOutputStream(localFile));
+ entity.writeTo(output);
+ } catch (HttpException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } finally {
+ try {
+ if (output != null)
+ output.close();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ File f = new File("/tmp/test");
+ Assert.assertEquals(f.length(), length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/c216990e/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java
----------------------------------------------------------------------
diff --git a/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java b/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java
index 8d73fcc..f565712 100644
--- a/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java
+++ b/engine/storage/src/org/apache/cloudstack/storage/to/TemplateTO.java
@@ -8,13 +8,12 @@ public class TemplateTO {
private final String uuid;
private final VolumeDiskType diskType;
private final ImageDataStoreTO imageDataStore;
- private final long size = 0;
+
public TemplateTO(TemplateInfo template) {
this.path = template.getPath();
this.uuid = template.getUuid();
this.diskType = template.getDiskType();
this.imageDataStore = new ImageDataStoreTO(template.getImageDataStore());
- // this.size = template.getVirtualSize();
}
public String getPath() {
@@ -32,8 +31,4 @@ public class TemplateTO {
public ImageDataStoreTO getImageDataStore() {
return this.imageDataStore;
}
-
- public long getSize() {
- return this.size;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/c216990e/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java
----------------------------------------------------------------------
diff --git a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java
index 435734c..8789bb7 100644
--- a/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java
+++ b/plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/XenServerStorageResource.java
@@ -18,6 +18,9 @@
*/
package com.cloud.hypervisor.xen.resource;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.util.HashMap;
@@ -36,6 +39,13 @@ import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import org.apache.xmlrpc.XmlRpcException;
@@ -67,31 +77,70 @@ public class XenServerStorageResource {
}
private long getTemplateSize(String url) {
- /*
- HttpGet method = new HttpGet(url);
- HttpClient client = new HttpClient();
+ HttpHead method = new HttpHead(url);
+ DefaultHttpClient client = new DefaultHttpClient();
try {
- int responseCode = client.executeMethod(method);
- if (responseCode != HttpStatus.SC_OK) {
- throw new CloudRuntimeException("http get returns error code:" + responseCode);
+ HttpResponse response = client.execute(method);
+ Header header = response.getFirstHeader("Content-Length");
+ if (header == null) {
+ throw new CloudRuntimeException("Can't get content-lenght header from :" + url);
}
- method.get
+ Long length = Long.parseLong(header.getValue());
+ return length;
} catch (HttpException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ throw new CloudRuntimeException("Failed to get template lenght", e);
} catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ throw new CloudRuntimeException("Failed to get template lenght", e);
+ } catch (NumberFormatException e) {
+ throw new CloudRuntimeException("Failed to get template lenght", e);
}
- */
- return 0;
}
- protected Answer directDownloadHttpTemplate(TemplateTO template, PrimaryDataStoreTO primarDataStore) {
+ private void downloadHttpToLocalFile(String destFilePath, String url) {
+ File destFile = new File(destFilePath);
+ if (!destFile.exists()) {
+ throw new CloudRuntimeException("dest file doesn't exist: " + destFilePath);
+ }
+
+ DefaultHttpClient client = new DefaultHttpClient();
+ HttpGet getMethod = new HttpGet(url);
+ HttpResponse response;
+ BufferedOutputStream output = null;
+ long length = 0;
+ try {
+ response = client.execute(getMethod);
+ HttpEntity entity = response.getEntity();
+ length = entity.getContentLength();
+ output = new BufferedOutputStream(new FileOutputStream(destFile));
+ entity.writeTo(output);
+ } catch (ClientProtocolException e) {
+ throw new CloudRuntimeException("Failed to download template", e);
+ } catch (IOException e) {
+ throw new CloudRuntimeException("Failed to download template", e);
+ } finally {
+ if (output != null) {
+ try {
+ output.close();
+ } catch (IOException e) {
+ throw new CloudRuntimeException("Failed to download template", e);
+ }
+ }
+ }
+
+ //double check the length
+ destFile = new File(destFilePath);
+ if (destFile.length() != length) {
+ throw new CloudRuntimeException("Download file length doesn't match: expected: " + length + ", actual: " + destFile.length());
+ }
+
+ }
+
+ protected Answer directDownloadHttpTemplate(CopyTemplateToPrimaryStorageCmd cmd, TemplateTO template, PrimaryDataStoreTO primarDataStore) {
String primaryStoreUuid = primarDataStore.getUuid();
Connection conn = hypervisorResource.getConnection();
SR poolsr = null;
VDI vdi = null;
+ boolean result = false;
try {
Set<SR> srs = SR.getByNameLabel(conn, primaryStoreUuid);
@@ -103,8 +152,8 @@ public class XenServerStorageResource {
vdir.nameLabel = "Base-Image-" + UUID.randomUUID().toString();
vdir.SR = poolsr;
vdir.type = Types.VdiType.USER;
-
- vdir.virtualSize = template.getSize();
+
+ vdir.virtualSize = getTemplateSize(template.getPath());
vdi = VDI.create(conn, vdir);
vdir = vdi.getRecord(conn);
@@ -114,27 +163,40 @@ public class XenServerStorageResource {
throw new CloudRuntimeException("Don't how to handle multiple pbds:" + pbds.size() + " for sr: " + poolsr.getUuid(conn));
}
PBD pbd = pbds.iterator().next();
- PBD.Record pbdRec = pbd.getRecord(conn);
Map<String, String> deviceCfg = pbd.getDeviceConfig(conn);
String pbdLocation = deviceCfg.get("location");
if (pbdLocation == null) {
throw new CloudRuntimeException("Can't get pbd: " + pbd.getUuid(conn) + " location");
}
- String vdiPath = pbdLocation + "/" + vdiLocation;
+ String vdiPath = pbdLocation + "/" + vdiLocation + ".vhd";
//download a url into vdipath
-
+ //downloadHttpToLocalFile(vdiPath, template.getPath());
+ hypervisorResource.callHostPlugin(conn, "vmopsStorage", "downloadTemplateFromUrl", "destPath", vdiPath, "srcUrl", template.getPath());
+ result = true;
+ return new CopyTemplateToPrimaryStorageAnswer(cmd, vdi.getUuid(conn));
} catch (BadServerResponse e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ s_logger.debug("Failed to download template", e);
} catch (XenAPIException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ s_logger.debug("Failed to download template", e);
} catch (XmlRpcException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ s_logger.debug("Failed to download template", e);
+ } catch (Exception e) {
+ s_logger.debug("Failed to download template", e);
+ } finally {
+ if (!result && vdi != null) {
+ try {
+ vdi.destroy(conn);
+ } catch (BadServerResponse e) {
+ s_logger.debug("Failed to cleanup newly created vdi");
+ } catch (XenAPIException e) {
+ s_logger.debug("Failed to cleanup newly created vdi");
+ } catch (XmlRpcException e) {
+ s_logger.debug("Failed to cleanup newly created vdi");
+ }
+ }
}
- return null;
+ return new Answer(cmd, false, "Failed to download template");
}
protected Answer execute(CopyTemplateToPrimaryStorageCmd cmd) {
@@ -142,7 +204,7 @@ public class XenServerStorageResource {
TemplateTO template = imageTO.getTemplate();
ImageDataStoreTO imageStore = template.getImageDataStore();
if (imageStore.getType().equalsIgnoreCase("http")) {
- return directDownloadHttpTemplate(template, imageTO.getPrimaryDataStore());
+ return directDownloadHttpTemplate(cmd, template, imageTO.getPrimaryDataStore());
} else {
return new Answer(cmd, false, "not implemented yet");
/*
http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/c216990e/scripts/vm/hypervisor/xenserver/storagePlugin
----------------------------------------------------------------------
diff --git a/scripts/vm/hypervisor/xenserver/storagePlugin b/scripts/vm/hypervisor/xenserver/storagePlugin
new file mode 100755
index 0000000..df1c340
--- /dev/null
+++ b/scripts/vm/hypervisor/xenserver/storagePlugin
@@ -0,0 +1,56 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Version @VERSION@
+#
+# A plugin for executing script needed by vmops cloud
+
+import os, sys, time
+import XenAPIPlugin
+sys.path.extend(["/opt/xensource/sm/", "/usr/local/sbin/", "/sbin/"])
+import base64
+import hostvmstats
+import socket
+import stat
+import tempfile
+import util
+import subprocess
+import zlib
+import urllib2
+from util import CommandException
+
+def echo(fn):
+ def wrapped(*v, **k):
+ name = fn.__name__
+ util.SMlog("#### VMOPS enter %s ####" % name )
+ res = fn(*v, **k)
+ util.SMlog("#### VMOPS exit %s ####" % name )
+ return res
+ return wrapped
+
+@echo
+def downloadTemplateFromUrl(session, args):
+ destPath = args["destPath"]
+ srcUrl = args["srcUrl"]
+ template = urllib2.urlopen(srcUrl)
+ destFile = open(destPath, "wb")
+ destFile.write(template.read())
+ destFile.close()
+
+if __name__ == "__main__":
+ XenAPIPlugin.dispatch({"downloadTemplateFromUrl": downloadTemplateFromUrl})