You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudstack.apache.org by ts...@apache.org on 2013/10/02 18:41:46 UTC
[16/50] [abbrv] git commit: updated refs/heads/marvin-refactor to
bbaf354
marvin_refactor: generation of code is in the generate module
CloudStack's API DSL is converted into marvin's modules using the
generate module.
Signed-off-by: Prasanna Santhanam <ts...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/8cdac504
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/8cdac504
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/8cdac504
Branch: refs/heads/marvin-refactor
Commit: 8cdac5040ba79356cf19492e2dc760e411300cc5
Parents: ff8dfe1
Author: Prasanna Santhanam <ts...@apache.org>
Authored: Sat Sep 7 10:57:11 2013 +0530
Committer: Prasanna Santhanam <ts...@apache.org>
Committed: Wed Oct 2 20:27:25 2013 +0530
----------------------------------------------------------------------
tools/marvin/marvin/codegenerator.py | 490 -----------------------
tools/marvin/marvin/cs_entity_generator.py | 384 ------------------
tools/marvin/marvin/generate/apitoentity.py | 131 ++++++
tools/marvin/marvin/generate/entity.py | 166 ++++++++
tools/marvin/marvin/generate/factory.py | 59 +++
tools/marvin/marvin/generate/linguist.py | 158 ++++++++
tools/marvin/marvin/generate/verbs.py | 27 ++
tools/marvin/marvin/generate/xmltoapi.py | 488 ++++++++++++++++++++++
8 files changed, 1029 insertions(+), 874 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/codegenerator.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/codegenerator.py b/tools/marvin/marvin/codegenerator.py
deleted file mode 100644
index 28ec4dd..0000000
--- a/tools/marvin/marvin/codegenerator.py
+++ /dev/null
@@ -1,490 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import xml.dom.minidom
-import json
-import os
-import sys
-import urllib2
-from argparse import ArgumentParser
-from textwrap import dedent
-from os import path
-from cs_entity_generator import write_entity_classes, get_actionable_entities
-
-
-
-class cmdParameterProperty(object):
- def __init__(self):
- self.name = None
- self.required = False
- self.entity = ""
- self.desc = ""
- self.type = "planObject"
- self.subProperties = []
-
-
-class cloudStackCmd(object):
- def __init__(self):
- self.name = ""
- self.desc = ""
- self.async = "false"
- self.entity = ""
- self.request = []
- self.response = []
-
-
-class codeGenerator(object):
- """
- Apache CloudStack- marvin python classes can be generated from the json
- returned by API discovery or from the xml spec of commands generated by
- the ApiDocWriter. This class provides helper methods for these uses.
- """
- space = ' '
- newline = '\n'
- cmdsName = []
-
- def __init__(self, outputFolder):
- self.cmd = None
- self.code = ""
- self.required = []
- self.subclass = []
- self.outputFolder = outputFolder
- lic = """\
- # Licensed to the Apache Software Foundation (ASF) under one
- # or more contributor license agreements. See the NOTICE file
- # distributed with this work for additional information
- # regarding copyright ownership. The ASF licenses this file
- # to you under the Apache License, Version 2.0 (the
- # "License"); you may not use this file except in compliance
- # with the License. You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing,
- # software distributed under the License is distributed on an
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- # KIND, either express or implied. See the License for the
- # specific language governing permissions and limitations
- # under the License.
-
- """
- self.license = dedent(lic)
-
- def addAttribute(self, attr, pro):
- value = pro.value
- if pro.required:
- self.required.append(attr)
- desc = pro.desc
- if desc is not None:
- self.code += self.space
- self.code += "''' " + pro.desc + " '''"
- self.code += self.newline
-
- self.code += self.space
- self.code += attr + " = " + str(value)
- self.code += self.newline
-
- def generateSubClass(self, name, properties):
- """Generate code for sub objects
- """
- subclass = 'class %s:\n'%name
- subclass += self.space + "def __init__(self):\n"
- for pro in properties:
- if pro.desc is not None:
- subclass += self.space + self.space + '""""%s"""\n' % pro.desc
- if len(pro.subProperties) > 0:
- subclass += self.space + self.space
- subclass += 'self.%s = []\n' % pro.name
- self.generateSubClass(pro.name, pro.subProperties)
- else:
- subclass += self.space + self.space
- subclass += 'self.%s = None\n' % pro.name
-
- self.subclass.append(subclass)
-
- def generateApiCmd(self, cmd):
- """Given an API cmd module generate the cloudstack cmd class in python
- """
- self.code = ""
- self.subclass = []
- self.cmd = cmd
- self.cmdsName.append(self.cmd.name)
- self.code = self.license
- self.code += self.newline
- self.code += '"""%s"""\n' % self.cmd.desc
- self.code += 'from baseCmd import *\n'
- self.code += 'from baseResponse import *\n'
- self.code += "class %sCmd (baseCmd):\n" % self.cmd.name
- self.code += self.space + "def __init__(self):\n"
- self.code += self.space + self.space
- self.code += 'self.isAsync = "%s"\n' % str(self.cmd.async).lower()
- self.code += self.space*2 + 'self.entity = "%s"\n' % self.cmd.entity
-
- for req in self.cmd.request:
- if req.desc is not None:
- self.code += self.space + self.space + '"""%s"""\n' % req.desc
- if req.required:
- self.code += self.space + self.space + '"""Required"""\n'
-
- value = "None"
- if req.type == "list" or req.type == "map":
- value = "[]"
-
- self.code += self.space + self.space
- self.code += 'self.%s = %s\n' % (req.name, value)
- if req.required:
- self.required.append(req.name)
-
- self.code += self.space + self.space + "self.required = ["
- for require in self.required:
- self.code += '"' + require + '",'
- self.code += "]\n"
- #print cmd.name, self.required
- self.required = []
-
- """generate response code"""
- subItems = {}
- self.code += self.newline
- self.code += 'class %sResponse (baseResponse):\n' % self.cmd.name
- self.code += self.space + "def __init__(self):\n"
- if len(self.cmd.response) == 0:
- self.code += self.space + self.space + "pass"
- else:
- for res in self.cmd.response:
- if res.desc is not None:
- self.code += self.space + self.space
- self.code += '"""%s"""\n' % res.desc
-
- if len(res.subProperties) > 0:
- self.code += self.space + self.space
- self.code += 'self.%s = []\n' % res.name
- self.generateSubClass(res.name, res.subProperties)
- else:
- self.code += self.space + self.space
- self.code += 'self.%s = None\n' % res.name
- self.code += self.newline
-
- for subclass in self.subclass:
- self.code += subclass + "\n"
- return self.code
-
- def write(self, out, modulename, code):
- """
- writes the generated code for `modulename` in the `out.cloudstackAPI` package
- @param out: absolute path where all api cmds are written to
- @param modulename: name of the module being written: eg: createNetwork
- @param code: Generated code
- @return: None
- """
- final_path = path.join(out, modulename)
- module = final_path + '.py' #eg: out/cloudstackAPI/modulename.py
- with open(module, "w") as fp:
- fp.write(code)
-
- def finalize(self):
- """Generate an api call
- """
- header = '"""Marvin TestClient for CloudStack"""\n'
- imports = "import copy\n"
- initCmdsList = '__all__ = ['
- body = ''
- body += "class CloudStackAPIClient(object):\n"
- body += self.space + 'def __init__(self, connection):\n'
- body += self.space + self.space + 'self.connection = connection\n'
- body += self.space + self.space + 'self._id = None\n'
- body += self.newline
-
- body += self.space + 'def __copy__(self):\n'
- body += self.space + self.space
- body += 'return CloudStackAPIClient(copy.copy(self.connection))\n'
- body += self.newline
-
- # The `id` property will be used to link the test with the cloud
- # resource being created
- # @property
- # def id(self):
- # return self._id
- #
- # @id.setter
- # def id(self, identifier):
- # self._id = identifier
-
- body += self.space + '@property' + self.newline
- body += self.space + 'def id(self):' + self.newline
- body += self.space*2 + 'return self._id' + self.newline
- body += self.newline
-
- body += self.space + '@id.setter' + self.newline
- body += self.space + 'def id(self, identifier):' + self.newline
- body += self.space*2 + 'self._id = identifier' + self.newline
- body += self.newline
-
- for cmdName in self.cmdsName:
- body += self.space
- body += 'def %s(self, command, method="GET"):\n' % cmdName
- body += self.space + self.space
- body += 'response = %sResponse()\n' % cmdName
- body += self.space + self.space
- body += 'response = self.connection.marvin_request(command,'
- body += ' response_type=response, method=method)\n'
- body += self.space + self.space + 'return response\n'
- body += self.newline
-
- imports += 'from %s import %sResponse\n' % (cmdName, cmdName)
- initCmdsList += '"%s",' % cmdName
-
- cloudstackApiClient = self.license + header + imports + body
- self.write(out=self.outputFolder, modulename='cloudstackAPIClient', code=cloudstackApiClient)
-
- '''generate __init__.py'''
- initCmdsList = self.license + initCmdsList + '"cloudstackAPIClient"]'
- self.write(out=self.outputFolder, modulename='__init__', code=initCmdsList)
-
- basecmd = self.license
- basecmd += '"""Base Command"""\n'
- basecmd += 'class baseCmd(object):\n'
- basecmd += self.space + 'pass\n'
- self.write(out=self.outputFolder, modulename='baseCmd', code=basecmd)
-
- baseResponse = self.license
- baseResponse += '"""Base Response"""\n'
- baseResponse += 'class baseResponse:\n'
- baseResponse += self.space + 'pass\n'
- self.write(out=self.outputFolder, modulename='baseResponse', code=baseResponse)
-
- def constructResponseFromXML(self, response):
- paramProperty = cmdParameterProperty()
- paramProperty.name = getText(response.getElementsByTagName('name'))
- paramProperty.desc = getText(response.
- getElementsByTagName('description'))
- if paramProperty.name.find('(*)') != -1:
- '''This is a list'''
- paramProperty.name = paramProperty.name.split('(*)')[0]
- argList = response.getElementsByTagName('arguments')[0].\
- getElementsByTagName('arg')
- for subresponse in argList:
- subProperty = self.constructResponseFromXML(subresponse)
- paramProperty.subProperties.append(subProperty)
- return paramProperty
-
- def loadCmdFromXML(self, dom):
- cmds = []
- for cmd in dom.getElementsByTagName("command"):
- csCmd = cloudStackCmd()
- csCmd.name = getText(cmd.getElementsByTagName('name'))
- assert csCmd.name
- if csCmd.name in ['login', 'logout']:
- continue
-
- csCmd.entity = getText(cmd.getElementsByTagName('entity'))
- assert csCmd.entity
-
- desc = getText(cmd.getElementsByTagName('description'))
- if desc:
- csCmd.desc = desc
-
- async = getText(cmd.getElementsByTagName('isAsync'))
- if async:
- csCmd.async = async
-
- argList = cmd.getElementsByTagName("request")[0].\
- getElementsByTagName("arg")
- for param in argList:
- paramProperty = cmdParameterProperty()
-
- paramProperty.name =\
- getText(param.getElementsByTagName('name'))
- assert paramProperty.name
-
- required = param.getElementsByTagName('required')
- if getText(required) == "true":
- paramProperty.required = getText(required)
-
- requestDescription = param.getElementsByTagName('description')
- if requestDescription:
- paramProperty.desc = getText(requestDescription)
-
- type = param.getElementsByTagName("type")
- if type:
- paramProperty.type = getText(type)
-
- csCmd.request.append(paramProperty)
-
- responseEle = cmd.getElementsByTagName("response")[0]
- for response in responseEle.getElementsByTagName("arg"):
- if response.parentNode != responseEle:
- continue
-
- paramProperty = self.constructResponseFromXML(response)
- csCmd.response.append(paramProperty)
-
- cmds.append(csCmd)
- return cmds
-
- def generateCodeFromXML(self, apiSpecFile):
- dom = xml.dom.minidom.parse(apiSpecFile)
- cmds = self.loadCmdFromXML(dom)
- for cmd in cmds:
- code = self.generateApiCmd(cmd)
- self.write(out=self.outputFolder, modulename=cmd.name, code=code)
- self.finalize()
-
- def constructResponseFromJSON(self, response):
- paramProperty = cmdParameterProperty()
- if 'name' in response:
- paramProperty.name = response['name']
- assert paramProperty.name, "%s has no property name" % response
-
- if 'description' in response:
- paramProperty.desc = response['description']
- if 'type' in response:
- if response['type'] in ['list', 'map', 'set']:
- #Here list becomes a subproperty
- if 'response' in response:
- for innerResponse in response['response']:
- subProperty =\
- self.constructResponseFromJSON(innerResponse)
- paramProperty.subProperties.append(subProperty)
- paramProperty.type = response['type']
- return paramProperty
-
- def loadCmdFromJSON(self, apiStream):
- if apiStream is None:
- raise Exception("No APIs found through discovery")
-
- jsonOut = apiStream.readlines()
- assert len(jsonOut) > 0
- apiDict = json.loads(jsonOut[0])
- if not 'listapisresponse' in apiDict:
- raise Exception("API discovery plugin response failed")
- if not 'count' in apiDict['listapisresponse']:
- raise Exception("Malformed api response")
-
- apilist = apiDict['listapisresponse']['api']
- cmds = []
- for cmd in apilist:
- csCmd = cloudStackCmd()
- if 'name' in cmd:
- csCmd.name = cmd['name']
- assert csCmd.name
- if csCmd.name in ['login', 'logout']:
- continue
-
- if cmd.has_key('entity'):
- csCmd.entity = cmd['entity']
- else:
- print csCmd.name + " has no entity"
-
- if 'description' in cmd:
- csCmd.desc = cmd['description']
-
- if 'isasync' in cmd:
- csCmd.async = cmd['isasync']
-
- for param in cmd['params']:
- paramProperty = cmdParameterProperty()
-
- if 'name' in param:
- paramProperty.name = param['name']
- assert paramProperty.name
-
- if 'required' in param:
- paramProperty.required = param['required']
-
- if 'description' in param:
- paramProperty.desc = param['description']
-
- if 'type' in param:
- paramProperty.type = param['type']
-
- csCmd.request.append(paramProperty)
-
- for response in cmd['response']:
- #FIXME: ExtractImage related APIs return empty dicts in response
- if len(response) > 0:
- paramProperty = self.constructResponseFromJSON(response)
- csCmd.response.append(paramProperty)
-
- cmds.append(csCmd)
- return cmds
-
- def generateCodeFromJSON(self, endpointUrl):
- """
- Api Discovery plugin returns the supported APIs of a CloudStack
- endpoint.
- @return: The classes in cloudstackAPI/ formed from api discovery json
- """
- if endpointUrl.find('response=json') >= 0:
- apiStream = urllib2.urlopen(endpointUrl)
- cmds = self.loadCmdFromJSON(apiStream)
- for cmd in cmds:
- code = self.generateApiCmd(cmd)
- self.write(out=self.outputFolder, modulename=cmd.name, code=code)
- self.finalize()
-
-
-def getText(elements):
- return elements[0].childNodes[0].nodeValue.strip()
-
-if __name__ == "__main__":
- parser = ArgumentParser()
- parser.add_argument("-o", "--output", dest="output",
- help="The path to the generated code entities, default\
- is .")
- parser.add_argument("-s", "--specfile", dest="spec",
- help="The path and name of the api spec xml file,\
- default is /etc/cloud/cli/commands.xml")
- parser.add_argument("-e", "--endpoint", dest="endpoint",
- help="The endpoint mgmt server (with open 8096) where\
- apis are discovered, default is localhost")
- parser.add_argument("-y", "--entity", dest="entity", action="store_true",
- help="Generate entity based classes")
-
- options = parser.parse_args()
-
- folder = "."
- if options.output is not None:
- folder = options.output
- apiModule = folder + "/cloudstackAPI"
- if not os.path.exists(apiModule):
- try:
- os.mkdir(apiModule)
- except:
- print "Failed to create folder %s, due to %s" % (apiModule,
- sys.exc_info())
- print parser.print_help()
- exit(2)
-
- apiSpecFile = "/etc/cloud/cli/commands.xml"
- if options.spec is not None:
- apiSpecFile = options.spec
- if not os.path.exists(apiSpecFile):
- print "the spec file %s does not exists" % apiSpecFile
- print parser.print_help()
- exit(1)
-
- cg = codeGenerator(apiModule)
- if options.spec is not None:
- cg.generateCodeFromXML(apiSpecFile)
- elif options.endpoint is not None:
- endpointUrl = 'http://%s:8096/client/api?command=listApis&\
-response=json' % options.endpoint
- cg.generateCodeFromJSON(endpointUrl)
-
- if options.entity:
- entities = get_actionable_entities()
- write_entity_classes(entities, "entity")
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/cs_entity_generator.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/cs_entity_generator.py b/tools/marvin/marvin/cs_entity_generator.py
deleted file mode 100644
index c3cda0c..0000000
--- a/tools/marvin/marvin/cs_entity_generator.py
+++ /dev/null
@@ -1,384 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-try:
- import inflect
-except ImportError:
- raise Exception("inflect installation not found. use pip install inflect to continue")
-
-grammar = ['create', 'list', 'delete', 'update', 'ldap', 'login', 'logout',
- 'enable', 'activate', 'disable', 'add', 'remove',
- 'attach', 'detach', 'associate', 'generate', 'assign',
- 'authorize', 'change', 'register', 'configure',
- 'start', 'restart', 'reboot', 'stop', 'reconnect',
- 'cancel', 'destroy', 'revoke', 'mark', 'reset',
- 'copy', 'extract', 'migrate', 'restore', 'suspend',
- 'get', 'query', 'prepare', 'deploy', 'upload', 'lock',
- 'disassociate', 'scale', 'dedicate', 'archive', 'find',
- 'recover', 'release', 'resize', 'revert', 'replace']
-
-LICENSE = """# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-
-def get_api_cmds():
- """ Returns the API cmdlet instances
-
- @return: instances of all the API commands exposed by CloudStack
- """
- namespace = {}
- execfile('cloudstackAPI/__init__.py', namespace)
- api_classes = __import__('cloudstackAPI', globals().update(namespace), fromlist=['*'], level=-1)
-
-
- cmdlist = map(
- lambda f: getattr(api_classes, f),
- filter(
- lambda t: t.startswith('__') == False,
- dir(api_classes)
- )
- )
- cmdlist = filter(
- lambda g: g is not None,
- cmdlist
- )
- clslist = map(
- lambda g: getattr(g, g.__name__.split('.')[-1] + 'Cmd'),
- filter(
- lambda h: h.__name__.split('.')[-1] not in ['baseCmd', 'baseResponse', 'cloudstackAPIClient'],
- cmdlist
- )
- )
- cmdlets = map(lambda t: t(), clslist)
- return cmdlets
-
-def singularize(word, num=0):
- """Use the inflect engine to make singular nouns of the entities
- @return: singular of `word`
- """
- inflector = inflect.engine()
- return inflector.singular_noun(word)
-
-def transform_api(api):
- """Brute-force transform for entities that don't match other transform rules
- """
- if api == 'ldapConfig':
- return 'configure', 'Ldap'
- elif api == 'ldapRemove':
- return 'remove', 'Ldap'
- elif api == 'login':
- return 'login', 'CloudStack'
- elif api == 'logout':
- return 'logout', 'CloudStack'
- return api, None
-
-def verb_adjust(api, entity):
- """
- Considers the prefix as the verb when no preposition transformers have been found in the API
- Only if the entity is contained in the API string
- """
- index = api.lower().find(entity.lower())
- if index > 0:
- return api[:index]
- else:
- return api
-
-def entity_adjust(entity):
- """
- Some entities are managed within CloudStack where they don't bear any resemblance to the API.
- Adjust such entities to a more sensible client side entity
-
- #BUG: Inflect engine returns IpAddress => IpAddres as singular
- """
- if entity == 'IpAddres' or entity == 'IPAddres':
- return 'IpAddress'
- elif entity == 'SecurityGroupIngres':
- return 'SecurityGroupIngress'
- elif entity == 'SecurityGroupEgres':
- return 'SecurityGroupEgress'
- elif entity == 'GuestO':
- return 'GuestOS'
- elif entity == 'LBStickines':
- return 'LBStickiness'
- #CloudStack denotes VirtualRouter as DomainRouter
- elif entity == 'DomainRouter':
- return 'VirtualRouter'
- #CloudStack denotes VirtualMachine as UserVm
- elif entity == 'UserVm':
- return 'VirtualMachine'
- #CloudStack denotes aliased NIC (with IP) as NicSecondaryIp
- elif entity == 'NicSecondaryIp':
- return 'Nic'
- elif entity == 'Site2SiteVpnConnection':
- return 'VpnConnection'
- elif entity == 'Site2SiteVpnGateway':
- return 'VpnGateway'
- elif entity == 'Site2SiteCustomerGateway':
- return 'VpnCustomerGateway'
- #Cloudstack returns Register entity for registerUserKeys
- elif entity == 'Register':
- return 'UserKeys'
- #Cloudstack maintains Template/ISO/Volume as single Image type
- #elif entity in ['Template', 'Volume']:
- # return 'Image'
- #extractImage returns an Extract response but is a property of Image
- elif entity == 'Extract':
- return 'Template'
- return entity
-
-def prepositon_transformer(preposition=None):
- """Returns a transformer for the entity if it has a doXPrepositionY style API
-
- @param entity: The entity eg: resetPasswordForVirtualMachine, preposition=For
- @return: transformed entity, Y is the entity and doX is the verb, eg: VirtualMachine, resetPassword
- """
- def transform_api_with_preposition(api):
- if api.find(preposition) > 0:
- if api[api.find(preposition) + len(preposition)].isupper():
- return api[:api.find(preposition)], api[api.find(preposition) + len(preposition):]
- return api, None
- return transform_api_with_preposition
-
-def skip_list():
- """APIs that we will not auto-generate
- """
- return ['cleanVMReservationsCmd']
-
-def get_transformers():
- """ List of transform rules as lambdas
- """
- transformers = [prepositon_transformer('Of'),
- prepositon_transformer('For'),
- prepositon_transformer('To'),
- prepositon_transformer('From'),
- prepositon_transformer('With'),
- transform_api]
- return transformers
-
-def get_verb_and_entity(cmd):
- """Break down the API cmd instance in to `verb` and `Entity`
- @return: verb, Entity tuple
- """
- api = cmd.__class__.__name__
- api = api.replace('Cmd', '')
- #apply known list of transformations
- matching_verbs = filter(lambda v: api.startswith(v), grammar)
- if len(matching_verbs) > 0:
- for transformer in get_transformers():
- if transformer(api)[1]:
- verb = transformer(api)[0]
- if cmd.entity:
- entity = singularize(cmd.entity) if singularize(cmd.entity) else cmd.entity
- else:
- entity = verb, \
- singularize(transformer(api)[1]) if singularize(transformer(api)[1]) else transformer(api)[1]
- entity = entity_adjust(entity)
- break
- else:
- verb = matching_verbs[0]
- entity = api.replace(verb, '')
- if cmd.entity:
- entity = singularize(cmd.entity) if singularize(cmd.entity) else cmd.entity
- else:
- entity = singularize(entity) if singularize(entity) else entity
- entity = entity_adjust(entity)
- verb = verb_adjust(api, entity)
- #print "%s => (verb, entity) = (%s, %s)" % (api, verb, entity)
- return verb, entity
- else:
- print "No matching verb, entity breakdown for api %s" % api
-
-def get_actionable_entities():
- """
- Inspect all entities and return a map of the Entity against the actions
- along with the required arguments to satisfy the action
-
- @return: Dictionary of Entity { "verb" : [required] }
- """
- cmdlets = sorted(filter(lambda api: api.__class__.__name__ not in skip_list(), get_api_cmds()),
- key=lambda k: get_verb_and_entity(k)[1])
-
- entities = {}
- for cmd in cmdlets:
- requireds = getattr(cmd, 'required')
- optionals = filter(lambda x: '__' not in x and x not in ['required', 'isAsync', 'entity'], dir(cmd))
- api = cmd.__class__.__name__
- if api in skip_list():
- continue
- verb, entity = get_verb_and_entity(cmd)
- if entity not in entities:
- entities[entity] = {}
- entities[entity][verb] = {}
- entities[entity][verb]['args'] = requireds
- entities[entity][verb]['optionals'] = optionals
- entities[entity][verb]['apimodule'] = cmd.__class__.__module__.split('.')[-1]
- entities[entity][verb]['apicmd'] = api
- print "Transformed %s APIs to %s entities successfully" % (len(cmdlets), len(entities)) \
- if len(cmdlets) > 0 \
- else "No transformations occurred"
- return entities
-
-
-def write_entity_classes(entities, module=None):
- """
- Writes the collected entity classes into `path`
-
- @param entities: dictionary of entities and the verbs acting on them
- @param module: top level module to the generated classes
- @return:
- """
- tabspace = ' '
- entitydict = {}
- for entity, actions in entities.iteritems():
- body = []
- imports = []
- imports.append('from cloudstackentity import CloudStackEntity')
- body.append('class %s(CloudStackEntity):' % entity)
- body.append('\n\n')
- body.append(tabspace + 'def __init__(self, items):')
- body.append(tabspace * 2 + 'self.__dict__.update(items)')
- body.append('\n')
- for action, details in actions.iteritems():
- imports.append('from marvin.cloudstackAPI import %s' % details['apimodule'])
- if action.startswith('create') or action.startswith('list') or action.startswith(
- 'register') or action.startswith('deploy'):
- body.append(tabspace + '@classmethod')
- if action not in ['create', 'deploy']:
- no_id_args = filter(lambda arg: arg!= 'id', details['args'])
- if len(no_id_args) > 0:
- body.append(tabspace + 'def %s(self, apiclient, %s, **kwargs):' % (
- action, ', '.join(list(set(no_id_args)))))
- else:
- body.append(tabspace + 'def %s(self, apiclient, **kwargs):' % (action))
- #TODO: Add docs for entity
- # doc to explain what possible args go into **kwargs
- body.append(tabspace * 2 + '"""Placeholder for docstring')
- body.append(tabspace * 2 + 'optional arguments (**kwargs): [%s]"""' % ', '.join(details['optionals']))
-
- body.append(tabspace * 2 + 'cmd = %(module)s.%(command)s()' % {"module": details["apimodule"],
- "command": details["apicmd"]})
- if 'id' in details['args']:
- body.append(tabspace * 2 + 'cmd.id = self.id')
- for arg in no_id_args:
- body.append(tabspace * 2 + 'cmd.%s = %s' % (arg, arg))
- body.append(tabspace * 2 + '[setattr(cmd, key, value) for key,value in kwargs.iteritems()]')
- body.append(tabspace * 2 + '%s = apiclient.%s(cmd)' % (entity.lower(), details['apimodule']))
- if action.startswith('list'):
- body.append(
- tabspace * 2 + 'return map(lambda e: %s(e.__dict__), %s) if %s and len(%s) > 0 else None' % (
- entity, entity.lower(), entity.lower(), entity.lower()))
- else:
- body.append(tabspace * 2 + 'return %s if %s else None' % (entity.lower(), entity.lower()))
- else:
- if len(details['args']) > 0:
- body.append(tabspace + 'def %s(cls, apiclient, %s, factory=None, **kwargs):' % (
- action, ', '.join(map(lambda arg: arg + '=None', list(set(details['args']))))))
- else:
- body.append(tabspace + 'def %s(cls, apiclient, factory=None, **kwargs):' % action)
- #TODO: Add docs for actions
- # doc to explain what possible args go into **kwargs
- body.append(tabspace * 2 + '"""Placeholder for docstring')
- body.append(tabspace * 2 + 'optional arguments (**kwargs): [%s]"""' % ', '.join(details['optionals']))
-
- body.append(tabspace * 2 + 'cmd = %(module)s.%(command)s()' % {"module": details["apimodule"],
- "command": details["apicmd"]})
- body.append(tabspace * 2 + 'if factory:')
- body.append(
- tabspace * 3 + '[setattr(cmd, factoryKey, factoryValue) for factoryKey, factoryValue in factory.__dict__.iteritems()]')
- if len(details["args"]) > 0:
- body.append(tabspace * 2 + 'else:')
- for arg in details["args"]:
- body.append(tabspace * 3 + "cmd.%s = %s" % (arg, arg))
- body.append(tabspace * 2 + '[setattr(cmd, key, value) for key, value in kwargs.iteritems()]')
- body.append(tabspace * 2 + '%s = apiclient.%s(cmd)' % (entity.lower(), details['apimodule']))
- body.append(
- tabspace * 2 + 'return %s(%s.__dict__) if %s else None' % (entity, entity.lower(), entity.lower()))
- body.append('\n')
-
- imports = '\n'.join(imports)
- body = '\n'.join(body)
- code = imports + '\n\n' + body
-
- entitydict[entity] = code
- write_entity_factory(entity, actions, module='factory')
- module_path = './' + '/'.join(module.split('.'))
- if not os.path.exists("%s" % module_path):
- os.makedirs("%s" % module_path)
- with open("%s/__init__.py" % (module_path), "w") as writer:
- writer.write(LICENSE)
- with open("%s/%s.py" % (module_path, entity.lower()), "w") as writer:
- writer.write(LICENSE)
- writer.write(code)
-
-def write_entity_factory(entity, actions, module=None):
- """Data factories for each entity
- """
-
- tabspace = ' '
- code = ''
- factory_defaults = []
- keys = actions.keys()
- for key in keys:
- if key.startswith('create'):
- factory_defaults.extend(actions[key]['args'])
- elif key.startswith('deploy'):
- factory_defaults.extend(actions[key]['args'])
- elif key.startswith('associate'):
- factory_defaults.extend(actions[key]['args'])
- elif key.startswith('register'):
- factory_defaults.extend(actions[key]['args'])
- else:
- continue
- #print '%s is not suitable for factory creation for entity %s' %(key, entity)
-
- factory_defaults = set(factory_defaults)
- module_path = './' + '/'.join(module.split('.'))
- if not os.path.exists(module_path):
- os.makedirs(module_path)
-
- code += 'from marvin.entity.%s import %s\n' % (entity.lower(), entity)
- code += 'from cloudstackbasefactory import CloudStackBaseFactory'
- code += '\n'
- code += 'class %sFactory(CloudStackBaseFactory):' % entity
- code += '\n\n'
- code += tabspace + 'FACTORY_FOR = %s\n\n' % entity
- for arg in factory_defaults:
- code += tabspace + '%s = None\n' % arg
- with open("%s/__init__.py" % (module_path), "w") as writer:
- writer.write(LICENSE)
- with open("%s/%s.py" % (module_path, entity.lower()), "w") as writer:
- writer.write(LICENSE)
- writer.write(code)
-
-if __name__ == '__main__':
- entities = get_actionable_entities()
- write_entity_classes(entities, 'entity')
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/generate/apitoentity.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/generate/apitoentity.py b/tools/marvin/marvin/generate/apitoentity.py
new file mode 100644
index 0000000..4a05ebf
--- /dev/null
+++ b/tools/marvin/marvin/generate/apitoentity.py
@@ -0,0 +1,131 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from marvin.generate.entity import Entity
+from marvin.generate.factory import Factory
+from marvin.generate.linguist import *
+
+LICENSE = """# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+
+def get_api_cmds():
+ """ Returns the API cmdlet instances
+
+ @return: instances of all the API commands exposed by CloudStack
+ """
+ namespace = {}
+ execfile('cloudstackAPI/__init__.py', namespace)
+ api_classes = __import__('cloudstackAPI', globals().update(namespace), fromlist=['*'], level=-1)
+
+
+ cmdlist = map(
+ lambda f: getattr(api_classes, f),
+ filter(
+ lambda t: t.startswith('__') == False,
+ dir(api_classes)
+ )
+ )
+ cmdlist = filter(
+ lambda g: g is not None,
+ cmdlist
+ )
+ clslist = map(
+ lambda g: getattr(g, g.__name__.split('.')[-1] + 'Cmd'),
+ filter(
+ lambda h: h.__name__.split('.')[-1] not in ['baseCmd', 'baseResponse', 'cloudstackAPIClient'],
+ cmdlist
+ )
+ )
+ cmdlets = map(lambda t: t(), clslist)
+ return cmdlets
+
+def get_entity_action_map():
+ """ Inspect cloudstack api and return a map of the Entity against the actions
+ along with the required arguments to make the action call
+
+ @return: Dictionary of Entity { "verb" : [required] }
+ eg: VirtualMachine { "deploy" : [templateid, serviceoffering, zoneid, etc] }
+ """
+ cmdlets = sorted(filter(lambda api: api.__class__.__name__ not in skip_list(), get_api_cmds()),
+ key=lambda k: get_verb_and_entity(k)[1])
+
+ entities = {}
+ for cmd in cmdlets:
+ requireds = getattr(cmd, 'required')
+ optionals = filter(lambda x: '__' not in x and x not in ['required', 'isAsync', 'entity'], dir(cmd))
+ api = cmd.__class__.__name__
+ if api in skip_list():
+ continue
+ verb, entity = get_verb_and_entity(cmd)
+ if entity not in entities:
+ entities[entity] = {}
+ entities[entity][verb] = {}
+ entities[entity][verb]['args'] = requireds
+ entities[entity][verb]['optionals'] = optionals
+ entities[entity][verb]['apimodule'] = cmd.__class__.__module__.split('.')[-1]
+ entities[entity][verb]['apicmd'] = api
+ print "Transformed %s APIs to %s entities successfully" % (len(cmdlets), len(entities)) \
+ if len(cmdlets) > 0 \
+ else "No transformations occurred"
+ return entities
+
+def write(entity_or_factory, module):
+ module_path = './' + '/'.join(module.split('.'))
+ if not os.path.exists("%s" % module_path):
+ os.makedirs("%s" % module_path)
+ with open("%s/__init__.py" % (module_path), "w") as writer:
+ writer.write(LICENSE)
+ with open("%s/%s.py" % (module_path, entity_or_factory.name.lower()), "w") as writer:
+ writer.write(LICENSE)
+ writer.write(entity_or_factory.__str__())
+
+def generate(entities):
+ """
+ Writes the collected entity classes
+
+ @param entities: dictionary of entities and the verbs acting on them
+ @return:
+ """
+ for entity, actions in entities.iteritems():
+ e = Entity()
+ f = Factory()
+
+ e.generate_entity(entity, actions)
+ f.generate_factory(entity, actions)
+
+ write(e, module='entity')
+ write(f, module='factory')
+
+if __name__ == '__main__':
+ entities = get_entity_action_map()
+ generate(entities)
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/generate/entity.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/generate/entity.py b/tools/marvin/marvin/generate/entity.py
new file mode 100644
index 0000000..76316ef
--- /dev/null
+++ b/tools/marvin/marvin/generate/entity.py
@@ -0,0 +1,166 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+class Entity(object):
+ """Defines a cloudstack entity and associated actions
+ """
+ def __init__(self):
+ self.name = None
+ self.classname = None
+ self.imports = []
+ self.methods = []
+ self.creator = None
+ self.enumerator = None
+ self.lines = []
+ self.tabspace = ' '
+ self.methods.append(self.update_method())
+
+ def __str__(self):
+ w = '\n'.join(self.imports)
+ w = w + '\n\n'
+ w = w + self.classname
+ w = w + '\n\n'
+ for m in self.methods:
+ if m.is_enumerator():
+ w = w + self.tabspace + '@classmethod'
+ w = w + '\n'
+ w = w + self.tabspace + m.signature
+ w = w + '\n'
+ for line in m.body:
+ w = w + self.tabspace + line
+ w = w + '\n'
+ w = w + '\n'
+ return w
+
+ def update_method(self):
+ """Defines the update builtin method on cloudstack entity
+ """
+ update = Method('__update__')
+ update.signature = 'def __update__(self, items):'
+ update.body = [self.tabspace + 'for key, value in items.iteritems():']
+ update.body.append(self.tabspace * 2 + 'setattr(self, key, value)')
+ update.body.append(self.tabspace + 'return self')
+ return update
+
+ def generate_entity(self, entity, actions):
+ self.imports.append('from cloudstackentity import CloudStackEntity')
+ self.name = entity
+ self.classname = 'class %s(CloudStackEntity):' % entity
+ for action, details in actions.iteritems():
+ self.imports.append('from marvin.cloudstackAPI import %s' % details['apimodule'])
+ m = Method(action)
+ self.methods.append(m)
+ #TODO: doc to explain what possible args go into **kwargs
+ m.docstring = 'Placeholder for docstring\n' + 'optional arguments (**kwargs): [%s]"""' % ', '.join(
+ details['optionals'])
+ if not m.is_creator():
+ # remove the id arg as id is the self (object) itself
+ no_id_args = filter(lambda arg: arg != 'id', details['args'])
+ if len(no_id_args) > 0: # at least one required non-id argument
+ m.signature = 'def %s(self, apiclient, %s, **kwargs):'\
+ % (action, ', '.join(list(set(no_id_args))))
+ else:
+ m.signature = 'def %s(self, apiclient, **kwargs):' % (action)
+ m.body.append(self.tabspace + 'cmd = %(module)s.%(command)s()' % {"module": details["apimodule"],
+ "command": details["apicmd"]})
+ if 'id' in details['args']:
+ m.body.append(self.tabspace + 'cmd.id = self.id')
+ for arg in no_id_args:
+ m.body.append(self.tabspace + 'cmd.%s = %s' % (arg, arg))
+ m.body.append(self.tabspace + '[setattr(cmd, key, value) for key, value in kwargs.iteritems()]')
+ m.body.append(self.tabspace + '%s = apiclient.%s(cmd)' % (entity.lower(), details['apimodule']))
+ if m.is_enumerator():
+ m.body.append(self.tabspace +
+ 'return map(lambda e: %s().__update__(e.__dict__), %s) '
+ 'if %s and len(%s) > 0 else None' % (
+ entity, entity.lower(), entity.lower(), entity.lower()))
+ else:
+ m.body.append(self.tabspace + 'return %s if %s else None' % (entity.lower(), entity.lower()))
+ else:
+ if len(details['args']) > 0: #has required arguments
+ m.signature = 'def __init__(self, apiclient=None, %s, factory=None, **kwargs):' % (
+ ', '.join(map(lambda arg: arg + '=None', list(set(details['args'])))))
+ else:
+ m.signature = 'def %s(cls, apiclient=None, factory=None, **kwargs):' % action
+
+ m.body.append(self.tabspace + 'if not apiclient:')
+ m.body.append(self.tabspace * 2 + 'self.__update__(kwargs)')
+ m.body.append(self.tabspace*2 + 'return')
+
+ m.body.append(self.tabspace + 'cmd = %(module)s.%(command)s()' % {"module": details["apimodule"],
+ "command": details["apicmd"]})
+ m.body.append(self.tabspace + 'if factory:')
+ m.body.append(
+ self.tabspace * 2 + '[setattr(cmd, factoryKey, factoryValue) for factoryKey, factoryValue in factory.__dict__.iteritems()]')
+ if len(details['args']) > 0: #has required arguments
+ m.body.append(self.tabspace + 'else:')
+ for arg in details['args']:
+ m.body.append(self.tabspace * 2 + "cmd.%s = %s" % (arg, arg))
+ m.body.append(self.tabspace + '[setattr(cmd, key, value) for key, value in kwargs.iteritems()]')
+ m.body.append(self.tabspace + '%s = apiclient.%s(cmd)' % (entity.lower(), details['apimodule']))
+ m.body.append(
+ self.tabspace + 'self.__update__(%s.__dict__) if %s else None' % (entity.lower(), entity.lower()))
+
+class Method(object):
+ """A method object defining action on an entity
+
+ - contains attributes, signature, docstring and body
+ """
+ def __init__(self, action):
+ self.action = action
+ self.docstring = None
+ self.signature = None
+ self.body = []
+
+ def is_creator(self):
+ """ Any action that results in the creation of the entity is an entity creator
+
+ eg: createNetwork, deployVirtualMachine or registerIso
+ @param action: action verb
+ @return: True if creator False otherwise
+ """
+ if self.action.startswith('create') \
+ or self.action.startswith('register') \
+ or self.action.startswith('deploy'):
+ return True
+ return False
+
+ def is_enumerator(self):
+ """ Any action that lists existing entities is an entity enumerator
+
+ eg: listXxx APIs
+ @param action: action verb
+ @return: True if enumerator False otherwise
+ """
+ if self.action.startswith('list'):
+ return True
+ return False
+
+
+class Creator(Method):
+ """A creator method - typically one that creates the entity
+ """
+ def __init__(self):
+ self.decorators = ["@classmethod"]
+
+
+class Enumerator(Method):
+ """An enumerator method - typically one that lists entities
+ """
+ def __init__(self):
+ self.decorators = ["@classmethod"]
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/generate/factory.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/generate/factory.py b/tools/marvin/marvin/generate/factory.py
new file mode 100644
index 0000000..d31f916
--- /dev/null
+++ b/tools/marvin/marvin/generate/factory.py
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class Factory(object):
+ """Defines a cloudstack factory object
+ """
+ def __init__(self):
+ self.name = None
+ self.code = None
+
+ def __str__(self):
+ return self.code
+
+ def generate_factory(self, entity, actions):
+ """Data factories for each entity
+ """
+
+ tabspace = ' '
+ code = ''
+ factory_defaults = []
+ keys = actions.keys()
+ for key in keys:
+ if key.startswith('create'):
+ factory_defaults.extend(actions[key]['args'])
+ elif key.startswith('deploy'):
+ factory_defaults.extend(actions[key]['args'])
+ elif key.startswith('associate'):
+ factory_defaults.extend(actions[key]['args'])
+ elif key.startswith('register'):
+ factory_defaults.extend(actions[key]['args'])
+ else:
+ continue
+ #print '%s is not suitable for factory creation for entity %s' %(key, entity)
+
+ factory_defaults = set(factory_defaults)
+ code += 'from marvin.entity.%s import %s\n' % (entity.lower(), entity)
+ code += 'from factory import Factory'
+ code += '\n\n'
+ code += 'class %sFactory(Factory):' % entity
+ code += '\n\n'
+ code += tabspace + 'FACTORY_FOR = %s\n\n' % entity
+ for arg in factory_defaults:
+ code += tabspace + '%s = None\n' % arg
+ self.name = entity
+ self.code = code
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/generate/linguist.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/generate/linguist.py b/tools/marvin/marvin/generate/linguist.py
new file mode 100644
index 0000000..82a7bfc
--- /dev/null
+++ b/tools/marvin/marvin/generate/linguist.py
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+try:
+ import inflect
+except ImportError:
+ raise Exception("inflect installation not found. use pip install inflect to continue")
+from marvin.generate.verbs import grammar
+
+def singularize(word, num=0):
+ """Use the inflect engine to make singular nouns of the entities
+ @return: singular of `word`
+ """
+ inflector = inflect.engine()
+ return inflector.singular_noun(word)
+
+def transform_api(api):
+ """Brute-force transform for entities that don't match other transform rules
+ """
+ if api == 'ldapConfig':
+ return 'configure', 'Ldap'
+ elif api == 'ldapRemove':
+ return 'remove', 'Ldap'
+ elif api == 'login':
+ return 'login', 'CloudStack'
+ elif api == 'logout':
+ return 'logout', 'CloudStack'
+ return api, None
+
+def verb_adjust(api, entity):
+ """
+ Considers the prefix as the verb when no preposition transformers have been found in the API
+ Only if the entity is contained in the API string
+ """
+ index = api.lower().find(entity.lower())
+ if index > 0:
+ return api[:index]
+ else:
+ return api
+
+def entity_adjust(entity):
+ """
+ Some entities are managed within CloudStack where they don't bear any resemblance to the API.
+ Adjust such entities to a more sensible client side entity
+
+ #BUG: Inflect engine returns IpAddress => IpAddres as singular
+ """
+ if entity == 'IpAddres' or entity == 'IPAddres':
+ return 'IpAddress'
+ elif entity == 'SecurityGroupIngres':
+ return 'SecurityGroupIngress'
+ elif entity == 'SecurityGroupEgres':
+ return 'SecurityGroupEgress'
+ elif entity == 'GuestO':
+ return 'GuestOS'
+ elif entity == 'LBStickines':
+ return 'LBStickiness'
+ #CloudStack denotes VirtualRouter as DomainRouter
+ elif entity == 'DomainRouter':
+ return 'VirtualRouter'
+ #CloudStack denotes VirtualMachine as UserVm
+ elif entity == 'UserVm':
+ return 'VirtualMachine'
+ #CloudStack denotes aliased NIC (with IP) as NicSecondaryIp
+ elif entity == 'NicSecondaryIp':
+ return 'Nic'
+ elif entity == 'Site2SiteVpnConnection':
+ return 'VpnConnection'
+ elif entity == 'Site2SiteVpnGateway':
+ return 'VpnGateway'
+ elif entity == 'Site2SiteCustomerGateway':
+ return 'VpnCustomerGateway'
+ #Cloudstack returns Register entity for registerUserKeys
+ elif entity == 'Register':
+ return 'UserKeys'
+ #Cloudstack maintains Template/ISO/Volume as single Image type
+ #elif entity in ['Template', 'Volume']:
+ # return 'Image'
+ #extractImage returns an Extract response but is a property of Image
+ elif entity == 'Extract':
+ return 'Template'
+ return entity
+
+def prepositon_transformer(preposition=None):
+ """Returns a transformer for the entity if it has a doXPrepositionY style API
+
+ @param entity: The entity eg: resetPasswordForVirtualMachine, preposition=For
+ @return: transformed entity, Y is the entity and doX is the verb, eg: VirtualMachine, resetPassword
+ """
+ def transform_api_with_preposition(api):
+ if api.find(preposition) > 0:
+ if api[api.find(preposition) + len(preposition)].isupper():
+ return api[:api.find(preposition)], api[api.find(preposition) + len(preposition):]
+ return api, None
+ return transform_api_with_preposition
+
+def skip_list():
+ """APIs that we will not auto-generate
+ """
+ return ['cleanVMReservationsCmd']
+
+def get_transformers():
+ """ List of transform rules as lambdas
+ """
+ transformers = [prepositon_transformer('Of'),
+ prepositon_transformer('For'),
+ prepositon_transformer('To'),
+ prepositon_transformer('From'),
+ prepositon_transformer('With'),
+ transform_api]
+ return transformers
+
+def get_verb_and_entity(cmd):
+ """Break down the API cmd instance in to `verb` and `Entity`
+ @return: verb, Entity tuple
+ """
+ api = cmd.__class__.__name__
+ api = api.replace('Cmd', '')
+ #apply known list of transformations
+ matching_verbs = filter(lambda v: api.startswith(v), grammar)
+ if len(matching_verbs) > 0:
+ for transformer in get_transformers():
+ if transformer(api)[1]:
+ verb = transformer(api)[0]
+ if cmd.entity:
+ entity = singularize(cmd.entity) if singularize(cmd.entity) else cmd.entity
+ else:
+ entity = verb, \
+ singularize(transformer(api)[1]) if singularize(transformer(api)[1]) else transformer(api)[1]
+ entity = entity_adjust(entity)
+ break
+ else:
+ verb = matching_verbs[0]
+ entity = api.replace(verb, '')
+ if cmd.entity:
+ entity = singularize(cmd.entity) if singularize(cmd.entity) else cmd.entity
+ else:
+ entity = singularize(entity) if singularize(entity) else entity
+ entity = entity_adjust(entity)
+ verb = verb_adjust(api, entity)
+ #print "%s => (verb, entity) = (%s, %s)" % (api, verb, entity)
+ return verb, entity
+ else:
+ print "No matching verb, entity breakdown for api %s" % api
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/generate/verbs.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/generate/verbs.py b/tools/marvin/marvin/generate/verbs.py
new file mode 100644
index 0000000..2051f2d
--- /dev/null
+++ b/tools/marvin/marvin/generate/verbs.py
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+grammar = ['create', 'list', 'delete', 'update', 'ldap', 'login', 'logout',
+ 'enable', 'activate', 'disable', 'add', 'remove',
+ 'attach', 'detach', 'associate', 'generate', 'assign',
+ 'authorize', 'change', 'register', 'configure',
+ 'start', 'restart', 'reboot', 'stop', 'reconnect',
+ 'cancel', 'destroy', 'revoke', 'mark', 'reset',
+ 'copy', 'extract', 'migrate', 'restore', 'suspend',
+ 'get', 'query', 'prepare', 'deploy', 'upload', 'lock',
+ 'disassociate', 'scale', 'dedicate', 'archive', 'find',
+ 'recover', 'release', 'resize', 'revert', 'replace']
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/8cdac504/tools/marvin/marvin/generate/xmltoapi.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/generate/xmltoapi.py b/tools/marvin/marvin/generate/xmltoapi.py
new file mode 100644
index 0000000..22a2786
--- /dev/null
+++ b/tools/marvin/marvin/generate/xmltoapi.py
@@ -0,0 +1,488 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import xml.dom.minidom
+import json
+import os
+import sys
+import urllib2
+from argparse import ArgumentParser
+from textwrap import dedent
+from os import path
+from marvin.generate.apitoentity import generate, get_entity_action_map
+
+class cmdParameterProperty(object):
+ def __init__(self):
+ self.name = None
+ self.required = False
+ self.entity = ""
+ self.desc = ""
+ self.type = "planObject"
+ self.subProperties = []
+
+
+class cloudStackCmd(object):
+ def __init__(self):
+ self.name = ""
+ self.desc = ""
+ self.async = "false"
+ self.entity = ""
+ self.request = []
+ self.response = []
+
+
+class codeGenerator(object):
+ """
+ Apache CloudStack- marvin python classes can be generated from the json
+ returned by API discovery or from the xml spec of commands generated by
+ the ApiDocWriter. This class provides helper methods for these uses.
+ """
+ space = ' '
+ newline = '\n'
+ cmdsName = []
+
+ def __init__(self, outputFolder):
+ self.cmd = None
+ self.code = ""
+ self.required = []
+ self.subclass = []
+ self.outputFolder = outputFolder
+ lic = """\
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+
+ """
+ self.license = dedent(lic)
+
+ def addAttribute(self, attr, pro):
+ value = pro.value
+ if pro.required:
+ self.required.append(attr)
+ desc = pro.desc
+ if desc is not None:
+ self.code += self.space
+ self.code += "''' " + pro.desc + " '''"
+ self.code += self.newline
+
+ self.code += self.space
+ self.code += attr + " = " + str(value)
+ self.code += self.newline
+
+ def generateSubClass(self, name, properties):
+ """Generate code for sub objects
+ """
+ subclass = 'class %s:\n'%name
+ subclass += self.space + "def __init__(self):\n"
+ for pro in properties:
+ if pro.desc is not None:
+ subclass += self.space + self.space + '""""%s"""\n' % pro.desc
+ if len(pro.subProperties) > 0:
+ subclass += self.space + self.space
+ subclass += 'self.%s = []\n' % pro.name
+ self.generateSubClass(pro.name, pro.subProperties)
+ else:
+ subclass += self.space + self.space
+ subclass += 'self.%s = None\n' % pro.name
+
+ self.subclass.append(subclass)
+
+ def generateApiCmd(self, cmd):
+ """Given an API cmd module generate the cloudstack cmd class in python
+ """
+ self.code = ""
+ self.subclass = []
+ self.cmd = cmd
+ self.cmdsName.append(self.cmd.name)
+ self.code = self.license
+ self.code += self.newline
+ self.code += '"""%s"""\n' % self.cmd.desc
+ self.code += 'from baseCmd import *\n'
+ self.code += 'from baseResponse import *\n'
+ self.code += "class %sCmd (baseCmd):\n" % self.cmd.name
+ self.code += self.space + "def __init__(self):\n"
+ self.code += self.space + self.space
+ self.code += 'self.isAsync = "%s"\n' % str(self.cmd.async).lower()
+ self.code += self.space*2 + 'self.entity = "%s"\n' % self.cmd.entity
+
+ for req in self.cmd.request:
+ if req.desc is not None:
+ self.code += self.space + self.space + '"""%s"""\n' % req.desc
+ if req.required:
+ self.code += self.space + self.space + '"""Required"""\n'
+
+ value = "None"
+ if req.type == "list" or req.type == "map":
+ value = "[]"
+
+ self.code += self.space + self.space
+ self.code += 'self.%s = %s\n' % (req.name, value)
+ if req.required:
+ self.required.append(req.name)
+
+ self.code += self.space + self.space + "self.required = ["
+ for require in self.required:
+ self.code += '"' + require + '",'
+ self.code += "]\n"
+ #print cmd.name, self.required
+ self.required = []
+
+ """generate response code"""
+ subItems = {}
+ self.code += self.newline
+ self.code += 'class %sResponse (baseResponse):\n' % self.cmd.name
+ self.code += self.space + "def __init__(self):\n"
+ if len(self.cmd.response) == 0:
+ self.code += self.space + self.space + "pass"
+ else:
+ for res in self.cmd.response:
+ if res.desc is not None:
+ self.code += self.space + self.space
+ self.code += '"""%s"""\n' % res.desc
+
+ if len(res.subProperties) > 0:
+ self.code += self.space + self.space
+ self.code += 'self.%s = []\n' % res.name
+ self.generateSubClass(res.name, res.subProperties)
+ else:
+ self.code += self.space + self.space
+ self.code += 'self.%s = None\n' % res.name
+ self.code += self.newline
+
+ for subclass in self.subclass:
+ self.code += subclass + "\n"
+ return self.code
+
+ def write(self, out, modulename, code):
+ """
+ writes the generated code for `modulename` in the `out.cloudstackAPI` package
+ @param out: absolute path where all api cmds are written to
+ @param modulename: name of the module being written: eg: createNetwork
+ @param code: Generated code
+ @return: None
+ """
+ final_path = path.join(out, modulename)
+ module = final_path + '.py' #eg: out/cloudstackAPI/modulename.py
+ with open(module, "w") as fp:
+ fp.write(code)
+
+ def finalize(self):
+ """Generate an api call
+ """
+ header = '"""Marvin TestClient for CloudStack"""\n'
+ imports = "import copy\n"
+ initCmdsList = '__all__ = ['
+ body = ''
+ body += "class CloudStackAPIClient(object):\n"
+ body += self.space + 'def __init__(self, connection):\n'
+ body += self.space + self.space + 'self.connection = connection\n'
+ body += self.space + self.space + 'self._id = None\n'
+ body += self.newline
+
+ body += self.space + 'def __copy__(self):\n'
+ body += self.space + self.space
+ body += 'return CloudStackAPIClient(copy.copy(self.connection))\n'
+ body += self.newline
+
+ # The `id` property will be used to link the test with the cloud
+ # resource being created
+ # @property
+ # def id(self):
+ # return self._id
+ #
+ # @id.setter
+ # def id(self, identifier):
+ # self._id = identifier
+
+ body += self.space + '@property' + self.newline
+ body += self.space + 'def id(self):' + self.newline
+ body += self.space*2 + 'return self._id' + self.newline
+ body += self.newline
+
+ body += self.space + '@id.setter' + self.newline
+ body += self.space + 'def id(self, identifier):' + self.newline
+ body += self.space*2 + 'self._id = identifier' + self.newline
+ body += self.newline
+
+ for cmdName in self.cmdsName:
+ body += self.space
+ body += 'def %s(self, command, method="GET"):\n' % cmdName
+ body += self.space + self.space
+ body += 'response = %sResponse()\n' % cmdName
+ body += self.space + self.space
+ body += 'response = self.connection.marvin_request(command,'
+ body += ' response_type=response, method=method)\n'
+ body += self.space + self.space + 'return response\n'
+ body += self.newline
+
+ imports += 'from %s import %sResponse\n' % (cmdName, cmdName)
+ initCmdsList += '"%s",' % cmdName
+
+ cloudstackApiClient = self.license + header + imports + body
+ self.write(out=self.outputFolder, modulename='cloudstackAPIClient', code=cloudstackApiClient)
+
+ '''generate __init__.py'''
+ initCmdsList = self.license + initCmdsList + '"cloudstackAPIClient"]'
+ self.write(out=self.outputFolder, modulename='__init__', code=initCmdsList)
+
+ basecmd = self.license
+ basecmd += '"""Base Command"""\n'
+ basecmd += 'class baseCmd(object):\n'
+ basecmd += self.space + 'pass\n'
+ self.write(out=self.outputFolder, modulename='baseCmd', code=basecmd)
+
+ baseResponse = self.license
+ baseResponse += '"""Base Response"""\n'
+ baseResponse += 'class baseResponse:\n'
+ baseResponse += self.space + 'pass\n'
+ self.write(out=self.outputFolder, modulename='baseResponse', code=baseResponse)
+
+ def constructResponseFromXML(self, response):
+ paramProperty = cmdParameterProperty()
+ paramProperty.name = getText(response.getElementsByTagName('name'))
+ paramProperty.desc = getText(response.
+ getElementsByTagName('description'))
+ if paramProperty.name.find('(*)') != -1:
+ '''This is a list'''
+ paramProperty.name = paramProperty.name.split('(*)')[0]
+ argList = response.getElementsByTagName('arguments')[0].\
+ getElementsByTagName('arg')
+ for subresponse in argList:
+ subProperty = self.constructResponseFromXML(subresponse)
+ paramProperty.subProperties.append(subProperty)
+ return paramProperty
+
+ def loadCmdFromXML(self, dom):
+ cmds = []
+ for cmd in dom.getElementsByTagName("command"):
+ csCmd = cloudStackCmd()
+ csCmd.name = getText(cmd.getElementsByTagName('name'))
+ assert csCmd.name
+ if csCmd.name in ['login', 'logout']:
+ continue
+
+ csCmd.entity = getText(cmd.getElementsByTagName('entity'))
+ assert csCmd.entity
+
+ desc = getText(cmd.getElementsByTagName('description'))
+ if desc:
+ csCmd.desc = desc
+
+ async = getText(cmd.getElementsByTagName('isAsync'))
+ if async:
+ csCmd.async = async
+
+ argList = cmd.getElementsByTagName("request")[0].\
+ getElementsByTagName("arg")
+ for param in argList:
+ paramProperty = cmdParameterProperty()
+
+ paramProperty.name =\
+ getText(param.getElementsByTagName('name'))
+ assert paramProperty.name
+
+ required = param.getElementsByTagName('required')
+ if getText(required) == "true":
+ paramProperty.required = getText(required)
+
+ requestDescription = param.getElementsByTagName('description')
+ if requestDescription:
+ paramProperty.desc = getText(requestDescription)
+
+ type = param.getElementsByTagName("type")
+ if type:
+ paramProperty.type = getText(type)
+
+ csCmd.request.append(paramProperty)
+
+ responseEle = cmd.getElementsByTagName("response")[0]
+ for response in responseEle.getElementsByTagName("arg"):
+ if response.parentNode != responseEle:
+ continue
+
+ paramProperty = self.constructResponseFromXML(response)
+ csCmd.response.append(paramProperty)
+
+ cmds.append(csCmd)
+ return cmds
+
+ def generateCodeFromXML(self, apiSpecFile):
+ dom = xml.dom.minidom.parse(apiSpecFile)
+ cmds = self.loadCmdFromXML(dom)
+ for cmd in cmds:
+ code = self.generateApiCmd(cmd)
+ self.write(out=self.outputFolder, modulename=cmd.name, code=code)
+ self.finalize()
+
+ def constructResponseFromJSON(self, response):
+ paramProperty = cmdParameterProperty()
+ if 'name' in response:
+ paramProperty.name = response['name']
+ assert paramProperty.name, "%s has no property name" % response
+
+ if 'description' in response:
+ paramProperty.desc = response['description']
+ if 'type' in response:
+ if response['type'] in ['list', 'map', 'set']:
+ #Here list becomes a subproperty
+ if 'response' in response:
+ for innerResponse in response['response']:
+ subProperty =\
+ self.constructResponseFromJSON(innerResponse)
+ paramProperty.subProperties.append(subProperty)
+ paramProperty.type = response['type']
+ return paramProperty
+
+ def loadCmdFromJSON(self, apiStream):
+ if apiStream is None:
+ raise Exception("No APIs found through discovery")
+
+ jsonOut = apiStream.readlines()
+ assert len(jsonOut) > 0
+ apiDict = json.loads(jsonOut[0])
+ if not 'listapisresponse' in apiDict:
+ raise Exception("API discovery plugin response failed")
+ if not 'count' in apiDict['listapisresponse']:
+ raise Exception("Malformed api response")
+
+ apilist = apiDict['listapisresponse']['api']
+ cmds = []
+ for cmd in apilist:
+ csCmd = cloudStackCmd()
+ if 'name' in cmd:
+ csCmd.name = cmd['name']
+ assert csCmd.name
+ if csCmd.name in ['login', 'logout']:
+ continue
+
+ if cmd.has_key('entity'):
+ csCmd.entity = cmd['entity']
+ else:
+ print csCmd.name + " has no entity"
+
+ if 'description' in cmd:
+ csCmd.desc = cmd['description']
+
+ if 'isasync' in cmd:
+ csCmd.async = cmd['isasync']
+
+ for param in cmd['params']:
+ paramProperty = cmdParameterProperty()
+
+ if 'name' in param:
+ paramProperty.name = param['name']
+ assert paramProperty.name
+
+ if 'required' in param:
+ paramProperty.required = param['required']
+
+ if 'description' in param:
+ paramProperty.desc = param['description']
+
+ if 'type' in param:
+ paramProperty.type = param['type']
+
+ csCmd.request.append(paramProperty)
+
+ for response in cmd['response']:
+ #FIXME: ExtractImage related APIs return empty dicts in response
+ if len(response) > 0:
+ paramProperty = self.constructResponseFromJSON(response)
+ csCmd.response.append(paramProperty)
+
+ cmds.append(csCmd)
+ return cmds
+
+ def generateCodeFromJSON(self, endpointUrl):
+ """
+ Api Discovery plugin returns the supported APIs of a CloudStack
+ endpoint.
+ @return: The classes in cloudstackAPI/ formed from api discovery json
+ """
+ if endpointUrl.find('response=json') >= 0:
+ apiStream = urllib2.urlopen(endpointUrl)
+ cmds = self.loadCmdFromJSON(apiStream)
+ for cmd in cmds:
+ code = self.generateApiCmd(cmd)
+ self.write(out=self.outputFolder, modulename=cmd.name, code=code)
+ self.finalize()
+
+
+def getText(elements):
+ return elements[0].childNodes[0].nodeValue.strip()
+
+if __name__ == "__main__":
+ parser = ArgumentParser()
+ parser.add_argument("-o", "--output", dest="output",
+ help="The path to the generated code entities, default\
+ is .")
+ parser.add_argument("-s", "--specfile", dest="spec",
+ help="The path and name of the api spec xml file,\
+ default is /etc/cloud/cli/commands.xml")
+ parser.add_argument("-e", "--endpoint", dest="endpoint",
+ help="The endpoint mgmt server (with open 8096) where\
+ apis are discovered, default is localhost")
+ parser.add_argument("-y", "--entity", dest="entity", action="store_true",
+ help="Generate entity based classes")
+
+ options = parser.parse_args()
+
+ folder = "."
+ if options.output is not None:
+ folder = options.output
+ apiModule = folder + "/cloudstackAPI"
+ if not os.path.exists(apiModule):
+ try:
+ os.mkdir(apiModule)
+ except:
+ print "Failed to create folder %s, due to %s" % (apiModule,
+ sys.exc_info())
+ print parser.print_help()
+ exit(2)
+
+ apiSpecFile = "/etc/cloud/cli/commands.xml"
+ if options.spec is not None:
+ apiSpecFile = options.spec
+ if not os.path.exists(apiSpecFile):
+ print "the spec file %s does not exists" % apiSpecFile
+ print parser.print_help()
+ exit(1)
+
+ cg = codeGenerator(apiModule)
+ if options.spec is not None:
+ cg.generateCodeFromXML(apiSpecFile)
+ elif options.endpoint is not None:
+ endpointUrl = 'http://%s:8096/client/api?command=listApis&\
+response=json' % options.endpoint
+ cg.generateCodeFromJSON(endpointUrl)
+
+ if options.entity:
+ entities = get_entity_action_map()
+ generate(entities, "entity")