You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ts...@apache.org on 2013/04/17 13:36:53 UTC
git commit: updated refs/heads/requestsformarvin to 20ce63f
Updated Branches:
refs/heads/requestsformarvin [created] 20ce63fcf
WIP refactor for cloudstackConnection
Moving to use requests and refactoring to include post data
Signed-off-by: Prasanna Santhanam <ts...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/20ce63fc
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/20ce63fc
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/20ce63fc
Branch: refs/heads/requestsformarvin
Commit: 20ce63fcf053f82a9b090f9a33c9fdc9f2897f54
Parents: fad55c5
Author: Prasanna Santhanam <ts...@apache.org>
Authored: Wed Apr 17 17:06:15 2013 +0530
Committer: Prasanna Santhanam <ts...@apache.org>
Committed: Wed Apr 17 17:06:15 2013 +0530
----------------------------------------------------------------------
tools/marvin/marvin/asyncJobMgr.py | 2 +-
tools/marvin/marvin/cloudstackConnection.py | 239 +++++++++++-----------
tools/marvin/marvin/cloudstackTestClient.py | 3 +-
tools/marvin/marvin/codegenerator.py | 4 +-
tools/marvin/marvin/deployDataCenter.py | 1 -
tools/marvin/marvin/jsonHelper.py | 7 +-
6 files changed, 130 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/asyncJobMgr.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/asyncJobMgr.py b/tools/marvin/marvin/asyncJobMgr.py
index 935bebe..40304fa 100644
--- a/tools/marvin/marvin/asyncJobMgr.py
+++ b/tools/marvin/marvin/asyncJobMgr.py
@@ -54,7 +54,7 @@ class workThread(threading.Thread):
try:
self.lock.acquire()
- result = self.connection.pollAsyncJob(job.jobId, job.responsecls).jobresult
+ result = self.connection.poll(job.jobId, job.responsecls).jobresult
except cloudstackException.cloudstackAPIException, e:
result = str(e)
finally:
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/cloudstackConnection.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py
index 1caeef3..06c846c 100644
--- a/tools/marvin/marvin/cloudstackConnection.py
+++ b/tools/marvin/marvin/cloudstackConnection.py
@@ -15,126 +15,133 @@
# specific language governing permissions and limitations
# under the License.
-import urllib2
+import requests
import urllib
-import httplib
import base64
import hmac
import hashlib
-import json
-import xml.dom.minidom
-import types
import time
-import inspect
import cloudstackException
from cloudstackAPI import *
import jsonHelper
+from requests import ConnectionError
+from requests import HTTPError
+from requests import Timeout
+from requests import RequestException
class cloudConnection(object):
- def __init__(self, mgtSvr, port=8096, apiKey = None, securityKey = None, asyncTimeout=3600, logging=None, protocol='http', path='/client/api'):
+ def __init__(self, mgtSvr, port=8096, apiKey=None, securityKey=None, asyncTimeout=3600, logging=None,
+ scheme='http', path='client/api'):
self.apiKey = apiKey
self.securityKey = securityKey
self.mgtSvr = mgtSvr
self.port = port
self.logging = logging
- if protocol != 'http' and protocol != 'https':
- raise ValueError("Protocol must be 'http' or 'https'.")
- else:
- self.protocol=protocol
self.path = path
- if port == 8096 or (self.apiKey == None and self.securityKey == None):
- self.auth = False
- else:
- self.auth = True
self.retries = 5
self.asyncTimeout = asyncTimeout
-
- def close(self):
- try:
- self.connection.close()
- except:
- pass
-
+ self.auth = True
+ if port == 8096 or \
+ (self.apiKey == None and self.securityKey == None):
+ self.auth = False
+ if scheme not in ['http', 'https']:
+ raise RequestException("Protocol must be HTTP")
+ self.protocol = scheme
+ self.baseurl = "%s://%s:%d/%s"%(self.protocol, self.mgtSvr, self.port, self.path)
+
def __copy__(self):
- return cloudConnection(self.mgtSvr, self.port, self.apiKey, self.securityKey, self.asyncTimeout, self.logging, self.protocol, self.path)
-
- def make_request_with_auth(self, command, requests={}):
- requests["command"] = command
- requests["apiKey"] = self.apiKey
- requests["response"] = "json"
- request = zip(requests.keys(), requests.values())
- request.sort(key=lambda x: str.lower(x[0]))
-
- requestUrl = "&".join(["=".join([r[0], urllib.quote_plus(str(r[1]))]) for r in request])
- hashStr = "&".join(["=".join([str.lower(r[0]), str.lower(urllib.quote_plus(str(r[1]))).replace("+", "%20")]) for r in request])
-
- sig = urllib.quote_plus(base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip())
- requestUrl += "&signature=%s"%sig
+ return cloudConnection(self.mgtSvr, self.port, self.apiKey, self.securityKey, self.asyncTimeout,
+ self.logging, self.protocol, self.path)
- try:
- self.connection = urllib2.urlopen("%s://%s:%d%s?%s"%(self.protocol, self.mgtSvr, self.port, self.path, requestUrl))
- if self.logging is not None:
- self.logging.debug("sending GET request: %s"%requestUrl)
- response = self.connection.read()
- if self.logging is not None:
- self.logging.info("got response: %s"%response)
- except IOError, e:
- if hasattr(e, 'reason'):
- if self.logging is not None:
- self.logging.critical("failed to reach %s because of %s"%(self.mgtSvr, e.reason))
- elif hasattr(e, 'code'):
- if self.logging is not None:
- self.logging.critical("server returned %d error code"%e.code)
- raise e
- except httplib.HTTPException, h:
- if self.logging is not None:
- self.logging.debug("encountered http Exception %s"%h.args)
- if self.retries > 0:
- self.retries = self.retries - 1
- self.make_request_with_auth(command, requests)
- else:
- self.retries = 5
- raise h
- else:
- return response
-
- def make_request_without_auth(self, command, requests={}):
- requests["command"] = command
- requests["response"] = "json"
- requests = zip(requests.keys(), requests.values())
- requestUrl = "&".join(["=".join([request[0], urllib.quote_plus(str(request[1]))]) for request in requests])
-
- self.connection = urllib2.urlopen("%s://%s:%d%s?%s"%(self.protocol, self.mgtSvr, self.port, self.path, requestUrl))
- if self.logging is not None:
- self.logging.debug("sending GET request without auth: %s"%requestUrl)
- response = self.connection.read()
- if self.logging is not None:
- self.logging.info("got response: %s"%response)
- return response
-
- def pollAsyncJob(self, jobId, response):
+ def poll(self, jobid, response):
+ """
+ polls the completion of a given jobid
+ @param jobid:
+ @param response:
+ @return:
+ """
cmd = queryAsyncJobResult.queryAsyncJobResultCmd()
- cmd.jobid = jobId
+ cmd.jobid = jobid
timeout = self.asyncTimeout
-
+
while timeout > 0:
- asyncResonse = self.make_request(cmd, response, True)
-
+ asyncResonse = self.marvin_request(cmd, response, True)
+
if asyncResonse.jobstatus == 2:
raise cloudstackException.cloudstackAPIException("asyncquery", asyncResonse.jobresult)
elif asyncResonse.jobstatus == 1:
return asyncResonse
-
+
time.sleep(5)
if self.logging is not None:
- self.logging.debug("job: %s still processing, will timeout in %ds"%(jobId, timeout))
+ self.logging.debug("job: %s still processing, will timeout in %ds"%(jobid, timeout))
timeout = timeout - 5
-
- raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout %s"%jobId)
-
- def make_request(self, cmd, response = None, raw=False):
- commandName = cmd.__class__.__name__.replace("Cmd", "")
- isAsync = "false"
+
+ raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout %s"%jobid)
+
+ def sign(self, payload):
+ """
+ signs a given request URL when the apiKey and secretKey are known
+
+ @param payload: dict of GET params to be signed
+ @return: the signature of the payload
+ """
+ params = zip(payload.keys(), map(lambda v: urllib.quote_plus(v), payload.values()))
+ params.sort(key=lambda k: str.lower(k[0]))
+ hashStr = "&".join(
+ ["=".join(
+ [str.lower(r[0]), str.lower(urllib.quote_plus(str(r[1]))).replace("+", "%20")]
+ ) for r in params]
+ )
+ signature = urllib.quote_plus(
+ base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip()
+ )
+ return signature
+
+ def request(self, command, auth=True, payload={}, data={}):
+ """
+ Makes requests on the `integration.api.port`
+ @param command: cloudstack API command name eg: deployVirtualMachineCommand
+ @param auth: Authentication (apikey,secretKey) => True, else False
+ @param payload: GET param data composed as a dictionary of key,value pairs
+ @param data: POST data as a dictionary
+ @return:
+ """
+ payload["command"] = command
+ payload["response"] = "json"
+
+ if auth:
+ payload["apiKey"] = self.apiKey
+ signature = self.sign(payload)
+ payload["signature"] = signature
+
+
+ try:
+ if data:
+ response = requests.get(self.baseurl, params=payload, data=data)
+ else:
+ response = requests.get(self.baseurl, params=payload)
+ except ConnectionError, c:
+ self.logging.debug("Connection refused. Reason: %s"%(self.baseurl, c))
+ raise c
+ except HTTPError, h:
+ self.logging.debug("Server returned error code: %s"%h)
+ raise h
+ except Timeout, t:
+ self.logging.debug("Connection timed out with %s"%t)
+ raise t
+ except RequestException,r:
+ self.logging.debug("Error returned by server %s"%r)
+ raise r
+ else:
+ return response
+
+ def sanitize_command(self, cmd):
+ """
+ Removes None values, Validates all required params are present
+ @param cmd: Cmd object eg: createPhysicalNetwork
+ @return:
+ """
requests = {}
required = []
for attribute in dir(cmd):
@@ -145,11 +152,11 @@ class cloudConnection(object):
required = getattr(cmd, attribute)
else:
requests[attribute] = getattr(cmd, attribute)
-
+
+ cmdname = cmd.__class__.__name__.replace("Cmd", "")
for requiredPara in required:
if requests[requiredPara] is None:
- raise cloudstackException.cloudstackAPIException(commandName, "%s is required"%requiredPara)
- '''remove none value'''
+ raise cloudstackException.cloudstackAPIException(cmdname, "%s is required"%requiredPara)
for param, value in requests.items():
if value is None:
requests.pop(param)
@@ -166,28 +173,30 @@ class cloudConnection(object):
for k,v in val.iteritems():
requests["%s[%d].%s"%(param,i,k)] = v
i = i + 1
-
- if self.logging is not None:
- self.logging.info("sending command: %s %s"%(commandName, str(requests)))
- result = None
+ return cmdname, isAsync, requests
+
+ def marvin_request(self, cmd, data={}, response_type=None):
+ """
+ Requester for marvin command objects
+ @param cmd: marvin's command from cloudstackAPI
+ @param data: any data to be sent in as POST
+ @param response_type: response type of the command in cmd
+ @param raw:
+ @return:
+ """
+ cmdname, isAsync, payload = self.sanitize_command(cmd)
+ self.logging.info("sending command: %s %s"%(cmdname, str(payload)))
if self.auth:
- result = self.make_request_with_auth(commandName, requests)
+ response = self.request(cmdname, auth=True, payload=payload, data=data)
else:
- result = self.make_request_without_auth(commandName, requests)
-
- if result is None:
- return None
-
- result = jsonHelper.getResultObj(result, response)
- if raw or isAsync == "false":
- return result
+ response = self.request(cmdname, auth=False, payload=payload, data=data)
+
+ self.logging.info("Request: %s Response: %s"%(response.url, response.text))
+ response = jsonHelper.getResultObj(response.json, response_type)
+
+ if isAsync == "false":
+ return response
else:
- asynJobId = result.jobid
- result = self.pollAsyncJob(asynJobId, response)
- return result.jobresult
-
-if __name__ == '__main__':
- xml = '<?xml version="1.0" encoding="ISO-8859-1"?><deployVirtualMachineResponse><virtualmachine><id>407</id><name>i-1-407-RS3</name><displayname>i-1-407-RS3</displayname><account>system</account><domainid>1</domainid><domain>ROOT</domain><created>2011-07-30T14:45:19-0700</created><state>Running</state><haenable>false</haenable><zoneid>1</zoneid><zonename>CA1</zonename><hostid>3</hostid><hostname>kvm-50-205</hostname><templateid>4</templateid><templatename>CentOS 5.5(64-bit) no GUI (KVM)</templatename><templatedisplaytext>CentOS 5.5(64-bit) no GUI (KVM)</templatedisplaytext><passwordenabled>false</passwordenabled><serviceofferingid>1</serviceofferingid><serviceofferingname>Small Instance</serviceofferingname><cpunumber>1</cpunumber><cpuspeed>500</cpuspeed><memory>512</memory><guestosid>112</guestosid><rootdeviceid>0</rootdeviceid><rootdevicetype>NetworkFilesystem</rootdevicetype><nic><id>380</id><networkid>203</networkid><netmask>255.255.255.0</netmask><gateway>65.19.181.1</gatew
ay><ipaddress>65.19.181.110</ipaddress><isolationuri>vlan://65</isolationuri><broadcasturi>vlan://65</broadcasturi><traffictype>Guest</traffictype><type>Direct</type><isdefault>true</isdefault><macaddress>06:52:da:00:00:08</macaddress></nic><hypervisor>KVM</hypervisor></virtualmachine></deployVirtualMachineResponse>'
- conn = cloudConnection(None)
-
- print conn.paraseReturnXML(xml, deployVirtualMachine.deployVirtualMachineResponse())
+ asyncJobId = response.jobid
+ response = self.poll(asyncJobId, response_type)
+ return response.jobresult
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/cloudstackTestClient.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py
index e4735e4..85552ed 100644
--- a/tools/marvin/marvin/cloudstackTestClient.py
+++ b/tools/marvin/marvin/cloudstackTestClient.py
@@ -111,7 +111,8 @@ class cloudstackTestClient(object):
apiKey = registerUserRes.apikey
securityKey = registerUserRes.secretkey
- newUserConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr, self.connection.port, apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging)
+ newUserConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr, self.connection.port,
+ apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging)
self.userApiClient = cloudstackAPIClient.CloudStackAPIClient(newUserConnection)
self.userApiClient.connection = newUserConnection
return self.userApiClient
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/codegenerator.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/codegenerator.py b/tools/marvin/marvin/codegenerator.py
index 0622e5d..b3b2a86 100644
--- a/tools/marvin/marvin/codegenerator.py
+++ b/tools/marvin/marvin/codegenerator.py
@@ -184,9 +184,9 @@ class codeGenerator:
body += "\n"
for cmdName in self.cmdsName:
- body += self.space + 'def %s(self,command):\n'%cmdName
+ body += self.space + 'def %s(self, command, postdata={}):\n'%cmdName
body += self.space + self.space + 'response = %sResponse()\n'%cmdName
- body += self.space + self.space + 'response = self.connection.make_request(command, response)\n'
+ body += self.space + self.space + 'response = self.connection.marvin_request(command, data=postdata, response_type=response)\n'
body += self.space + self.space + 'return response\n'
body += '\n'
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/deployDataCenter.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py
index 2e270a7..d358789 100644
--- a/tools/marvin/marvin/deployDataCenter.py
+++ b/tools/marvin/marvin/deployDataCenter.py
@@ -407,7 +407,6 @@ class deployDataCenters():
logging=self.testClientLogger)
if mgt.apiKey is None:
apiKey, securityKey = self.registerApiKey()
- self.testClient.close()
self.testClient = \
cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, 8080, \
apiKey, securityKey, \
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/jsonHelper.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/jsonHelper.py b/tools/marvin/marvin/jsonHelper.py
index 652cce0..37363bc 100644
--- a/tools/marvin/marvin/jsonHelper.py
+++ b/tools/marvin/marvin/jsonHelper.py
@@ -19,7 +19,6 @@ import cloudstackException
import json
import inspect
from cloudstackAPI import *
-import pdb
class jsonLoader:
'''The recursive class for building and representing objects with.'''
@@ -113,12 +112,8 @@ def finalizeResultObj(result, responseName, responsecls):
return result
else:
return result
-
-
-
+
def getResultObj(returnObj, responsecls=None):
- returnObj = json.loads(returnObj)
-
if len(returnObj) == 0:
return None
responseName = filter(lambda a: a!=u'cloudstack-version', returnObj.keys())[0]