You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gump.apache.org by le...@apache.org on 2005/04/27 21:26:39 UTC
svn commit: r165030 - in /gump/branches/Gump3/pygump/python/gump: config.py
model/util.py plugins/buildintelligence.py
Author: leosimons
Date: Wed Apr 27 12:26:38 2005
New Revision: 165030
URL: http://svn.apache.org/viewcvs?rev=165030&view=rev
Log:
Start on more intelligent build algorithm.
It turns out that the 'multicastplugin' (or whatever it is that acts as visitor to the walker) basically is the right place to implement smarter algorithms. I.e. anything above and beyond the depth-first graph walking goes into the visitor. For now I've started by extending multicastplugin. We might wish to rename that, break it out into its own module, and remove it a little from 'public' view. I'm not sure yet.
The code I'm committing doesn't work at the moment (well it seems like it does, it just skips all build steps since there was no module that was updated), doesn't have tests, and should probably be refactored. I'm checking it in so we can have a discussion about the whole multicast business. If I'm right, we'll want several more of those, all implementing different algorithm ideas.
Stefano, if you're reading, this is the place to do the graph magic :-)
Added:
gump/branches/Gump3/pygump/python/gump/plugins/buildintelligence.py
Modified:
gump/branches/Gump3/pygump/python/gump/config.py
gump/branches/Gump3/pygump/python/gump/model/util.py
Modified: gump/branches/Gump3/pygump/python/gump/config.py
URL: http://svn.apache.org/viewcvs/gump/branches/Gump3/pygump/python/gump/config.py?rev=165030&r1=165029&r2=165030&view=diff
==============================================================================
--- gump/branches/Gump3/pygump/python/gump/config.py (original)
+++ gump/branches/Gump3/pygump/python/gump/config.py Wed Apr 27 12:26:38 2005
@@ -124,10 +124,10 @@
from gump.plugins.builder import ScriptBuilderPlugin
plugins.append(ScriptBuilderPlugin(config.paths_work,buildlog))
- from gump.plugins.java.builder import ClasspathPlugin
- plugins.append(ClasspathPlugin(config.paths_work,buildlog))
- from gump.plugins.java.builder import AntPlugin
- plugins.append(AntPlugin(config.paths_work,buildlog))
+ #from gump.plugins.java.builder import ClasspathPlugin
+ #plugins.append(ClasspathPlugin(config.paths_work,buildlog))
+ #from gump.plugins.java.builder import AntPlugin
+ #plugins.append(AntPlugin(config.paths_work,buildlog))
post_process_plugins = []
# TODO: append more plugins here...
@@ -276,11 +276,13 @@
def get_plugin(config):
"""Provide a Plugin implementation."""
from gump.plugins import MulticastPlugin
+ #from gump.plugins.buildintelligence import MoreEfficientMulticastPlugin
(pre_process_plugins, plugins, post_process_plugins) = get_plugins(config)
error_handler = get_error_handler(config)
return (MulticastPlugin(pre_process_plugins, error_handler),
+ #MoreEfficientMulticastPlugin(plugins, error_handler),
MulticastPlugin(plugins, error_handler),
MulticastPlugin(post_process_plugins, error_handler))
Modified: gump/branches/Gump3/pygump/python/gump/model/util.py
URL: http://svn.apache.org/viewcvs/gump/branches/Gump3/pygump/python/gump/model/util.py?rev=165030&r1=165029&r2=165030&view=diff
==============================================================================
--- gump/branches/Gump3/pygump/python/gump/model/util.py (original)
+++ gump/branches/Gump3/pygump/python/gump/model/util.py Wed Apr 27 12:26:38 2005
@@ -22,6 +22,8 @@
from os.path import abspath
from os.path import join
+from gump.model import ModelObject, Error, Dependency, CvsModule, SvnModule
+
def get_project_directory(workdir, project):
"""Determine the base directory for a project."""
return get_module_directory(workdir, project.module)
@@ -37,3 +39,100 @@
def get_workspace_directory(workdir, workspace):
"""Determine the base directory for a workspace."""
return abspath(join(workdir,workspace.name))
+
+def mark_failure(model_element, cause):
+ """Mark a model element as "failed"."""
+ assert isinstance(model_element, ModelObject)
+ model_element.failed = True
+ if not hasattr(model_element, "failure_cause"):
+ model_element.failure_cause = []
+ model_element.failure_cause.append(cause)
+
+def check_failure(model_element):
+ """Determine whether a model element has "failed"."""
+ assert isinstance(model_element, ModelObject)
+ return getattr(model_element, "failed", False)
+
+def get_failure_causes(model_element):
+ """Get an array of "failure" "causes"."""
+ assert isinstance(model_element, ModelObject)
+ assert getattr(model_element, "failed", False)
+ assert len(getattr(model_element, "failure_cause", [])) > 0
+
+ return model_element.failure_cause
+
+def get_cause_for_cause(cause):
+ """Get whatever "caused" this "cause" (the "parent" cause).
+
+ Returns only the *first* parent cause, not all of them. Returns
+ None if there's no "parent" "cause"."""
+ if not isinstance(cause, ModelObject) \
+ or not len(getattr(model_element, "failure_cause", [])) > 0:
+ return None
+
+ return cause.failure_cause[0]
+
+def get_root_cause(model_element):
+ """Digs into a model element "blame" stack to find the "root" "cause".
+
+ Returns an array containing a trace of all the different causes, starting
+ with the main cause and ending with the "root" cause."""
+ assert isinstance(model_element, ModelObject)
+ assert isinstance(model_element, ModelObject)
+
+ cause = model_element.failure_cause[0]
+ trace = [cause]
+ while True:
+ parent_cause = get_cause_for_cause(cause)
+ if parent_cause:
+ if isinstance(parent_cause, Dependency):
+ parent_cause = parent_cause.dependency
+
+ if parent_cause in trace:
+ # something is seriously screwed up here -- there's
+ # a cyclic "cause". Now what? We'll assume that whatever
+ # code is calling us is not really prepared to handle
+ # something like that. For now we'll raise an exception.
+ # I don't think we'll see this in practice -- its a
+ # programming error within gump
+ raise Error, "Impossible to find root cause as there's a cycle!"
+ cause = parent_cause
+ trace.append(cause)
+ else:
+ break
+
+ return trace
+
+def mark_skip(model_element):
+ """Mark a model element as "should be skipped"."""
+ model_element.skip = True
+
+def check_skip(model_element):
+ """Determine whether a model element "should be skipped"."""
+ return getattr(model_element, "skip", False)
+
+def mark_whether_module_was_updated(module):
+ """Mark a module as "updated" if it was."""
+ # checkout means we did in fact update
+ if hasattr(module, "update_type"):
+ if module.update_type == UPDATE_TYPE_CHECKOUT and not module.failed:
+ module.was_updated = True
+
+ # no checkout, so an update. Might have had no effect. Check.
+ if hasattr(module, "update_log") and not hasattr(module, "was_updated"):
+ log = module.update_log.strip()
+
+ # a non empty cvs log message means we did in fact update
+ if isinstance(module, CvsModule):
+ if len(log) > 0:
+ module.was_updated = True
+
+ # a svn log message always has a line "At revision ...."
+ # if there's other lines with content it means we did in
+ # fact update
+ if isinstance(module, SvnModule):
+ log.replace('\r\n', '\n')
+ log.replace('\r', '\n')
+ lines = log.split('\n')
+ if len(lines) > 1:
+ module.was_updated = True
Added: gump/branches/Gump3/pygump/python/gump/plugins/buildintelligence.py
URL: http://svn.apache.org/viewcvs/gump/branches/Gump3/pygump/python/gump/plugins/buildintelligence.py?rev=165030&view=auto
==============================================================================
--- gump/branches/Gump3/pygump/python/gump/plugins/buildintelligence.py (added)
+++ gump/branches/Gump3/pygump/python/gump/plugins/buildintelligence.py Wed Apr 27 12:26:38 2005
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+# Copyright 2004-2005 The Apache Software Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains several modules which can be plugged into the pygump engine.
+
+A plugin is an instance of a class that extends AbstractPlugin. See the
+documentation for AbstractPlugin to learn more about the contracts
+surrounding plugins.
+"""
+
+__copyright__ = "Copyright (c) 2004-2005 The Apache Software Foundation"
+__license__ = "http://www.apache.org/licenses/LICENSE-2.0"
+
+from gump.model import ModelObject, CvsModule
+from gump.model.util import mark_failure, check_failure, mark_skip, check_skip
+from gump.model.util import mark_whether_module_was_updated
+
+from gump.plugins import MulticastPlugin, OptimisticLoggingErrorHandler
+from gump.plugins.updater import UPDATE_TYPE_CHECKOUT
+from gump.plugins.updater import UPDATE_TYPE_UPDATE
+
+class MoreEfficientMulticastPlugin(MulticastPlugin):
+ """MulticastPlugin that implements a more efficient build algorithm.
+
+ This plugin detects "failure" for a particular step in the build, and when
+ it does, it sometimes skips a few steps.
+
+ The following rules are implemented:
+ - if a module fails to update, the projects it contains are not built
+ - if there were no changes to the module since the previous run, the projects
+ it contains are not built
+ - if a project fails to build, none of its dependees are built
+
+ If an element "fails", its "failed" property will be set to "True" and an
+ array named "failure_cause" will be created pointing to the elements that
+ "caused" them to fail.
+ """
+ #def __init__(self, plugin_list):
+ # MulticastPlugin.__init__(self, plugin_list, OptimisticLoggingErrorHandler())
+
+ def visit_module(self, module):
+ # run the delegates
+ try:
+ for visitor in self.list:
+ visitor._visit_module(module)
+ except:
+ (type, value, traceback) = sys.exc_info()
+ self.error_handler.handle(visitor, module, type, value, traceback)
+ mark_failure(module, module.exceptions[len(module.exceptions)-1])
+
+ # check for update errors
+ if getattr(module, "update_exit_status", False):
+ mark_failure(module, module)
+ # if module update failed, don't try and attempt to build contained
+ # projects either.
+ for project in module.projects.values():
+ mark_failure(project, module)
+
+ # check for changes
+ mark_whether_module_was_updated(module)
+ if not getattr(module, "was_updated", False):
+ for project in module.projects.values():
+ mark_skip(project)
+
+ def visit_project(self, project):
+ # check for dependencies that failed to build
+ for relationship in project.dependencies:
+ if check_failure(relationship.dependency):
+ mark_failure(project, relationship)
+
+ # don't build if its not going to do any good
+ if not check_failure(project) and not check_skip(project):
+ try:
+ for visitor in self.list:
+ visitor._visit_project(project)
+ except:
+ (type, value, traceback) = sys.exc_info()
+ self.error_handler.handle(visitor, project, type, value, traceback)
+ mark_failure(project, project.exceptions[len(project.exceptions)-1])
+
+ # blame commands that went awry
+ for command in project.commands:
+ if getattr(command, "build_exit_status", False):
+ mark_failure(command, command)
+ mark_failure(project, command)