You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gump.apache.org by le...@apache.org on 2005/01/04 13:30:26 UTC
svn commit: r124101 - in gump/branches/Dec04MajorCleanup/pygump/python/gump: engine util
Author: leosimons
Date: Tue Jan 4 04:30:23 2005
New Revision: 124101
URL: http://svn.apache.org/viewcvs?view=rev&rev=124101
Log:
* Move VFS stuff and stream handling into a new IO package.
* some more cleanups and renames as coding conventions are falling out
* repository parsing for the Objectifier
Added:
gump/branches/Dec04MajorCleanup/pygump/python/gump/util/io.py
- copied, changed from r124040, gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/vfs.py
Removed:
gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/vfs.py
Modified:
gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py
gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py
Modified: gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py?view=diff&rev=124101&p1=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py&r1=124100&p2=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py&r2=124101
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py (original)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/__init__.py Tue Jan 4 04:30:23 2005
@@ -17,8 +17,8 @@
"""This module contains the main controller that runs pygump.
The function main() is called from the main.main() function, which construcs
-the neccessary helpers for an Engine instance that is then told to perform a
-run.
+the neccessary helpers for an _Engine instance that is then told to perform a
+run().
The engine utilizes several submodules that provide pieces of its
functionality. Each of these modules usually exports a single class which has
@@ -44,9 +44,12 @@
from xml import dom
from xml.dom import minidom
+from gump.util.io import open_file_or_stream
+
def main(settings):
- """
- Controls the big pygump beast. This function is called from the main.main()
+ """Controls the big pygump beast.
+
+ This function is called from the main.main()
method once the environment has been analyzed, parsed, fully set
up, and checked for correctness. In other words, once the beastie is
ready to roll.
@@ -61,30 +64,29 @@
_banner(settings.version)
# get engine config
- config = get_config(settings)
+ config = _get_config(settings)
# get engine dependencies
- log = get_logger(config.log_level, "engine")
- db = get_db(config)
+ log = _get_logger(config.log_level, "engine")
+ db = _get_db(config)
vfsdir = os.path.join(config.paths_work, "vfs-cache")
if not os.path.isdir(vfsdir):
os.mkdir(vfsdir);
- vfs = get_vfs(config.paths_metadata, vfsdir)
+ vfs = _get_vfs(config.paths_metadata, vfsdir)
+
+ modeller_log = _get_logger(config.log_level, "modeller")
+ modeller_loader = _get_modeller_loader(modeller_log, vfs)
+ modeller_objectifier = _get_modeller_objectifier(modeller_log)
- modeller_log = get_logger(config.log_level, "modeller")
mergefile = os.path.join(config.paths_work, "merge.xml")
dropfile = os.path.join(config.paths_work, "dropped.xml")
- modeller_loader = get_modeller_loader(modeller_log, vfs, mergefile, dropfile)
- modeller_objectifier = get_modeller_objectifier(modeller_log)
-
+
# create engine
- engine = Engine(config, log, db, modeller_loader, modeller_objectifier)
+ engine = _Engine(log, db, modeller_loader, modeller_objectifier, config.paths_workspace, mergefile, dropfile)
# run it
- engine.initialize()
engine.run()
- engine.dispose()
def _banner(version):
@@ -103,15 +105,15 @@
### FACTORY METHODS
###
-def get_config(settings):
- """
- Convert the settings object (the Values retrieved from the OptionsParser)
- into something more specific. The reason we put this function and the Config
+def _get_config(settings):
+ """Convert the settings object into something more specific.
+
+ The reason we put this function and the Config
class in between is that we can change gump internals while keeping the CLI
interface the same more easily, with the integration point being isolated to
this method and the Config class definition.
"""
- config = Config()
+ config = _Config()
if settings.debug:
config.log_level = logging.DEBUG
@@ -148,26 +150,31 @@
return config
-def get_logger(level, name):
+def _get_logger(level, name):
+ """Provide a logging implementation for the given level and name."""
logging.basicConfig()
log = logging.getLogger(name)
log.setLevel(level)
return log
-def get_db(config):
+def _get_db(config):
+ """Provide a database implementation."""
from gump.util.mysql import Database
db = Database(config) #TODO!
return db
-def get_vfs(filesystem_root, cache_dir):
- from gump.engine.vfs import VFS
+def _get_vfs(filesystem_root, cache_dir):
+ """Provide a VFS implementation."""
+ from gump.util.io import VFS
return VFS(filesystem_root, cache_dir)
-def get_modeller_loader(log, vfs=None, mergefile=None, dropfile=None):
+def _get_modeller_loader(log, vfs=None, mergefile=None, dropfile=None):
+ """Provide a Loader implementation."""
from gump.engine.modeller import Loader
return Loader(log, vfs, mergefile, dropfile)
-def get_modeller_objectifier(log):
+def _get_modeller_objectifier(log):
+ """Provide a Objectifier implementation."""
from gump.engine.modeller import Objectifier
return Objectifier(log)
@@ -175,12 +182,9 @@
### Classes
###
-class Config:
+class _Config:
def __getattr__(self,name):
- """
- Some config values are calculated at runtime from other values
- if they're not especially set.
- """
+ """Calculate missing settings from other settings at runtime."""
if name == 'debug':
return self.loglevel >= logging.DEBUG
if name == 'paths_pygump':
@@ -193,36 +197,43 @@
# unknown, raise error
raise AttributeError, name
-class Engine:
- """
- This is the core of the pygump application.
- """
+class _Engine:
+ """This is the core of the core of the pygump application."""
- def __init__(self, config, log, db, workspace_loader, workspace_objectifier):
- """
- Store all config and dependencies as properties.
+ def __init__(self, log, db, workspace_loader, workspace_objectifier, workspace, merge_to=None, drop_to=None):
+ """Store all config and dependencies as properties.
+
+ Arguments
+ - log -- the log to write debug and error messages to.
+ - db -- the database to store all activity in.
+ - workspace_loader -- the component providing the dom tree.
+ - workspace_objectifier -- the component transforming the dom into
+ object form
+
+ - workspace -- the resource containing the workspace xml.
+ - merge_to -- the resource to write the merged workspace xml to.
+ - drop_to -- the resource to write the dropped projects xml to.
"""
- self.config = config
self.log = log
self.db = db
self.workspace_loader = workspace_loader
self.workspace_objectifier = workspace_objectifier
-
- def initialize(self):
- """
- Perform pre-run initialization.
- """
- pass
-
+
+ self.workspace = open_file_or_stream(workspace,'r')
+ self.merge_to = open_file_or_stream(merge_to,'w')
+ self.drop_to = open_file_or_stream(drop_to,'w')
+
def run(self):
- """
- Perform a run.
- """
+ """Perform a run."""
try:
# 1) merge workspace into big DOM tree
- (dom, dropped_nodes) = self.workspace_loader.get_workspace_tree(self.config.paths_workspace)
+ (dom, dropped_nodes) = self.workspace_loader.get_workspace_tree(self.workspace)
+
+ # 2) write the merged tree out to a new xml file
+ self._write_merge_files(dom, dropped_nodes)
+
# 2) convert that DOM tree into python objects
- workspace = self.workspace_objectifier.get_workspace(dom)
+ workspace = self.workspace_objectifier.get_workspace(dom.documentElement)
# 3) store those objects in the database
self.store_workspace(self.workspace) #TODO
# 4) determine the tasks to perform
@@ -235,8 +246,22 @@
except:
self.log.exception("Fatal error during run!")
- def dispose(self):
- """
- End a run.
+ def _write_merge_files(self, domtree, dropped_nodes):
+ """Write the fully resolved DOM tree to a file.
+
+ Also writes an XML file detailing any projects and modules that were
+ dropped because of a HREF resolution issue.
"""
- pass
+ if self.merge_to:
+ self.merge_to.write( domtree.toprettyxml() )
+ self.merge_to.close()
+
+ if self.drop_to and len(dropped_nodes) > 0:
+ from xml import dom
+ impl = dom.getDOMImplementation()
+ dropdoc = impl.createDocument(None, "dropped-projects-and-modules", None)
+ dropdocroot = dropdoc.documentElement
+ for node in dropped_nodes:
+ dropdocroot.appendChild(node)
+ self.drop_to.write( dropdoc.toprettyxml() )
+ self.drop_to.close()
Modified: gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py?view=diff&rev=124101&p1=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py&r1=124100&p2=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py&r2=124101
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py (original)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/modeller.py Tue Jan 4 04:30:23 2005
@@ -27,13 +27,6 @@
from gump.model import *
-import types # TODO: move into utility module
-try:
- _StringTypes = [types.StringType, types.UnicodeType]
-except AttributeError:
- _StringTypes = [types.StringType]
-
-
class ModellerError(Exception):
"""Generic error thrown for all internal Modeller module exceptions."""
pass
@@ -87,38 +80,15 @@
- vfs -- the virtual file system to use for resolving hrefs. May
be None only if the workspace to load does not contain
any hrefs.
- - mergefile -- the full path to the file to write the merged
- workspace xml to. If None, no such file will be
- written. Alternatively can be a stream.
- - dropfile -- the full path to the file to write the xml
- describing dropped projects to. If None, no such
- file will be written. Alternatively can be a
- stream.
"""
self.log = log
self.vfs = vfs
- self.mergestream = self._open_file_or_stream(merge_file_or_stream)
- self.dropstream = self._open_file_or_stream(drop_file_or_stream)
- def _open_file_or_stream(self, file_or_stream): # TODO: move into utility module
- """Utility for accepting both filenames and streams.
-
- If the argument is a string, attempts to open the specified file and
- return a reference to the open file. Otherwise, returns the argument.
- """
- if type(file_or_stream) in _StringTypes:
- return open(file_or_stream, 'w')
- else:
- return file_or_stream
-
def get_workspace_tree(self, workspace):
"""Parse the provided workspace, then resolve all hrefs.
Returns a tuple with the parsed workspace as a dom Element, and all
the Nodes that were dropped because of problems as a list.
-
- The provided workspace argument can either be a stream or a path to
- a file.
"""
# get root <workspace/> element
wsdom = minidom.parse(workspace)
@@ -132,11 +102,8 @@
# contents of blah.xml
self._resolve_hrefs_in_workspace(ws, dropped_nodes)
- # write the merged xml tree to a file
- self._write_merge_files(wsdom, dropped_nodes)
-
# the combined results
- return (ws, dropped_nodes)
+ return (wsdom, dropped_nodes)
def _resolve_hrefs_in_workspace(self, ws, dropped_nodes):
"""Redirects to _resolve_hrefs_in_children."""
@@ -302,87 +269,49 @@
parent = parent.parentNode
if not parent:
return None
-
- def _write_merge_files(self, wsdom, dropped_nodes):
- """Write the fully resolved DOM tree to a file.
-
- Also writes an XML file detailing any projects and modules that were
- dropped because of a HREF resolution issue.
- """
- if self.mergestream:
- self.mergestream.write( wsdom.toprettyxml() )
- self.mergestream.close()
-
- if self.dropstream and len(dropped_nodes) > 0:
- impl = dom.getDOMImplementation()
- dropdoc = impl.createDocument(None, "dropped-projects-and-modules", None)
- dropdocroot = dropdoc.documentElement
- for node in dropped_nodes:
- dropdocroot.appendChild(node)
- self.dropstream.write( dropdoc.toprettyxml() )
- self.dropstream.close()
class Objectifier:
"""Turns DOM workspace into Pythonified workspace."""
def __init__(self, log):
+ """Store all settings and dependencies as properties."""
self.log = log
def get_workspace(self, dom):
- raise RuntimeError, "not implemented!" # TODO
-
+ """Transforms a workspace xml document into object form.
+
+ Travels the entire document tree, converting everything it finds."""
+
workspace = self._create_workspace(dom)
self._create_repositories(workspace, dom)
- self._create_modules(workspace, dom)
- self._create_projects(workspace, dom)
+
+ raise RuntimeError, "not implemented!" # TODO
+ #self._create_modules(workspace.repositories, dom)
+ #self._create_projects(workspace, dom)
+ return workspace
def _create_workspace(self, root):
- workspace = Workspace(root.getAttribute('name'))
-
+ return Workspace(root.getAttribute('name'))
def _create_repositories(self, workspace, root):
- repository_definitions = self._find_repository_definitions(root)
+ """Creates all repositories and adds them to the workspace."""
- undefined = []
+ repository_definitions = self._find_repository_definitions(root)
+ undefined = [] # store repositories that are referenced by name
for repository_definition in repository_definitions:
- if not repository_definition.hasChildNodes(): # hope it gets defined later
+ if not repository_definition.hasChildNodes():
+ # we hope this repository gets fully defined later, skip for now
if not repository.getAttribute("name"):
+ #TODO: implement recovery
raise ModellerError, "Encountered a repository without a name!"
undefined.append(repository_definition)
continue
- name = repository_definition.getAttribute("name")
- title = None
- try: title = self._find_element_text(repository_definition, "title")
- except: pass
-
- home_page = None
- try: home_page = self._find_element_text(repository_definition, "home-page")
- except: pass
-
- cvsweb = None
- try: cvsweb = self._find_element_text(repository_definition, "cvsweb")
- except:
- try: cvsweb = self._find_element_text(repository_definition, "web")
- except: pass
-
- redistributable = False
- if repository_definition.getElementsByTagName("redistributable").length > 0:
- redistributable = True
-
- repository = None
-
- type = repository_definition.getAttribute("type").upper()
- if type == "CVS":
- repository = _create_cvs_repository(workspace, name, title, home_page, cvsweb, redistributable, repository_definition)
- elif type == "SVN":
- repository = _create_svn_repository(workspace, name, title, home_page, cvsweb, redistributable, repository_definition)
- else:
- raise ModellerError, "Unknown repository type '%s' for repository '%s'" % (type, name)
-
- workspace.repositories[name] = repository
+ repository = self._create_repository(workspace, repository_definition)
+ # TODO: detect overrides here
+ workspace.repositories[repository.name] = repository
# TODO: add support for maven repository definitions here as found
# inside maven project.xml files...
@@ -397,6 +326,42 @@
undefined = None # clean up just to be sure...
+ def _create_repository(self, workspace, repository_definition):
+ name = repository_definition.getAttribute("name")
+ self.log.debug("Converting repository definition '%s' into object form." % name)
+
+ # parse the attributes and elements common to all repositories
+ title = None
+ try: title = self._find_element_text(repository_definition, "title")
+ except: pass
+
+ home_page = None
+ try: home_page = self._find_element_text(repository_definition, "home-page")
+ except: pass
+
+ cvsweb = None
+ try: cvsweb = self._find_element_text(repository_definition, "cvsweb")
+ except:
+ try: cvsweb = self._find_element_text(repository_definition, "web")
+ except: pass
+
+ redistributable = False
+ if repository_definition.getElementsByTagName("redistributable").length > 0:
+ redistributable = True
+
+ # now delegate to _create methods for specific repositories to do the rest
+ repository = None
+ type = repository_definition.getAttribute("type").upper()
+ if type == "CVS":
+ repository = self._create_cvs_repository(workspace, name, title, home_page, cvsweb, redistributable, repository_definition)
+ elif type == "SVN":
+ repository = self._create_svn_repository(workspace, name, title, home_page, cvsweb, redistributable, repository_definition)
+ else:
+ raise ModellerError, "Unknown repository type '%s' for repository '%s'" % (type, name)
+
+ return repository
+
+
def _create_cvs_repository(self, workspace, name, title, home_page, cvsweb, redistributable, repository_definition):
hostname = self._find_element_text(repository_definition, "hostname")
path = self._find_element_text(repository_definition, "path")
@@ -422,6 +387,28 @@
cvsweb = cvsweb,
redistributable = False,
method = CVS_METHOD_PSERVER,
+ user = user,
+ password = password)
+ return repository
+
+ def _create_svn_repository(self, workspace, name, title, home_page, cvsweb, redistributable, repository_definition):
+ url = self._find_element_text(repository_definition, "url")
+
+ user = None
+ try: user = self._find_element_text(repository_definition, "user")
+ except: pass
+
+ password = None
+ try: password = self._find_element_text(repository_definition, "password")
+ except: pass
+
+ repository = SvnRepository(workspace,
+ name,
+ url,
+ title = title,
+ home_page = home_page,
+ cvsweb = cvsweb,
+ redistributable = False,
user = user,
password = password)
return repository
Deleted: /gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/vfs.py
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/vfs.py?view=auto&rev=124100
==============================================================================
Copied: gump/branches/Dec04MajorCleanup/pygump/python/gump/util/io.py (from r124040, gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/vfs.py)
Url: http://svn.apache.org/viewcvs/gump/branches/Dec04MajorCleanup/pygump/python/gump/util/io.py?view=diff&rev=124101&p1=gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/vfs.py&r1=124040&p2=gump/branches/Dec04MajorCleanup/pygump/python/gump/util/io.py&r2=124101
==============================================================================
--- gump/branches/Dec04MajorCleanup/pygump/python/gump/engine/vfs.py (original)
+++ gump/branches/Dec04MajorCleanup/pygump/python/gump/util/io.py Tue Jan 4 04:30:23 2005
@@ -29,10 +29,34 @@
import string
import urllib
import urlparse
+import StringIO
+import types
+
+try:
+ _StringTypes = [types.StringType, types.UnicodeType]
+except AttributeError:
+ _StringTypes = [types.StringType]
+
+def open_file_or_stream(file_or_stream, mode='r'):
+ """Utility for accepting both filenames and streams.
+
+ If the argument is a string, attempt to use it as a filename,
+ open the specified file and return a reference to the open file. If it is
+ not a valid filename, wrap the string as a stream-like object and return it.
+
+ If the argument is not a string, returns the string.
+ """
+ if type(file_or_stream) in _StringTypes:
+ #try:
+ return open(file_or_stream, mode)
+ #except:
+ # StringIO.StringIO(file_or_stream)
+ else:
+ return file_or_stream
class Error(Exception):
- """Generic error thrown for all internal VFS module exceptions."""
+ """Generic error thrown for all internal I/O module exceptions."""
pass