You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2014/01/07 15:43:46 UTC
git commit: [#6992] Refactor bulk_export task some using Boundaries
concepts
Updated Branches:
refs/heads/master 18e8da135 -> d2baf30fc
[#6992] Refactor bulk_export task some using Boundaries concepts
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d2baf30f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d2baf30f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d2baf30f
Branch: refs/heads/master
Commit: d2baf30fcf0ab455050725b4461cd7798cfd329f
Parents: 18e8da1
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Thu Dec 19 20:22:58 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Jan 6 15:16:51 2014 +0000
----------------------------------------------------------------------
Allura/allura/tasks/export_tasks.py | 153 ++++++++++++++-----------------
Allura/allura/tests/test_tasks.py | 51 +++--------
2 files changed, 85 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d2baf30f/Allura/allura/tasks/export_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/export_tasks.py b/Allura/allura/tasks/export_tasks.py
index 2f286d0..3fd9499 100644
--- a/Allura/allura/tasks/export_tasks.py
+++ b/Allura/allura/tasks/export_tasks.py
@@ -44,89 +44,76 @@ def bulk_export(tools, filename=None, send_email=True):
'''
# it's very handy to use c.* within a @task,
# but let's be explicit and keep it separate from the main code
- return _bulk_export(c.project, tools, c.user, filename, send_email)
-
-
-def _bulk_export(project, tools, user, filename=None, send_email=True):
- export_filename = filename or project.bulk_export_filename()
- export_path = create_export_dir(project, export_filename)
- not_exported_tools = []
- for tool in tools or []:
- app = project.app_instance(tool)
- if not app:
- log.info('Can not load app for %s mount point. Skipping.' % tool)
- not_exported_tools.append(tool)
- continue
- if not app.exportable:
- log.info('Tool %s is not exportable. Skipping.' % tool)
- not_exported_tools.append(tool)
- continue
- log.info('Exporting %s...' % tool)
+ return BulkExport().process(c.project, tools, c.user, filename, send_email)
+
+
+class BulkExport(object):
+ def process(self, project, tools, user, filename=None, send_email=True):
+ export_filename = filename or project.bulk_export_filename()
+ export_path = self.get_export_path(project.bulk_export_path(), export_filename)
+ if not os.path.exists(export_path):
+ os.makedirs(export_path)
+ apps = [project.app_instance(tool) for tool in tools]
+ exportable = self.filter_exportable(apps)
+ results = [self.export(export_path, app) for app in exportable]
+ exported = self.filter_successful(results)
+ if exported:
+ zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename))
+ shutil.rmtree(export_path)
+
+ if not user:
+ log.info('No user. Skipping notification.')
+ return
+ if not send_email:
+ return
+
+ tmpl = g.jinja2_env.get_template('allura:templates/mail/bulk_export.html')
+ instructions = tg.config.get('bulk_export_download_instructions', '')
+ instructions = instructions.format(
+ project=project.shortname,
+ filename=export_filename,
+ c=c,
+ )
+ exported_names = [a.config.options.mount_point for a in exported]
+ tmpl_context = {
+ 'instructions': instructions,
+ 'project': project,
+ 'tools': exported_names,
+ 'not_exported_tools': list(set(tools) - set(exported_names)),
+ }
+ email = {
+ 'sender': unicode(tg.config['forgemail.return_path']),
+ 'fromaddr': unicode(tg.config['forgemail.return_path']),
+ 'reply_to': unicode(tg.config['forgemail.return_path']),
+ 'message_id': h.gen_message_id(),
+ 'destinations': [unicode(user._id)],
+ 'subject': u'Bulk export for project %s completed' % project.shortname,
+ 'text': tmpl.render(tmpl_context),
+ }
+ mail_tasks.sendmail.post(**email)
+
+ def get_export_path(self, export_base_path, export_filename):
+ """Create temporary directory for export files"""
+ # Name temporary directory after project shortname,
+ # thus zipdir() will use proper prefix inside the archive.
+ tmp_dir_suffix = os.path.splitext(export_filename)[0]
+ path = os.path.join(export_base_path, tmp_dir_suffix)
+ return path
+
+ def filter_exportable(self, apps):
+ return [app for app in apps if app and app.exportable]
+
+ def export(self, export_path, app):
+ tool = app.config.options.mount_point
+ json_file = os.path.join(export_path, '%s.json' % tool)
try:
- json_file = os.path.join(export_path, '%s.json' % tool)
with open(json_file, 'w') as f:
app.bulk_export(f)
- except:
- log.error('Something went wrong during export of %s' % tool, exc_info=True)
- not_exported_tools.append(tool)
- continue
-
- if tools and len(not_exported_tools) < len(tools):
- # If that fails, we need to let it fail
- # there won't be a valid zip file for the user to get.
- zip_and_cleanup(export_path, export_filename)
- else:
- log.error('Nothing to export')
- return None
-
- if not user:
- log.info('No user. Skipping notification.')
- return
- if not send_email:
- return
-
- tmpl = g.jinja2_env.get_template('allura:templates/mail/bulk_export.html')
- instructions = tg.config.get('bulk_export_download_instructions', '')
- instructions = instructions.format(
- project=project.shortname,
- filename=export_filename,
- c=c,
- )
- tmpl_context = {
- 'instructions': instructions,
- 'project': project,
- 'tools': list(set(tools) - set(not_exported_tools)),
- 'not_exported_tools': not_exported_tools,
- }
- email = {
- 'sender': unicode(tg.config['forgemail.return_path']),
- 'fromaddr': unicode(tg.config['forgemail.return_path']),
- 'reply_to': unicode(tg.config['forgemail.return_path']),
- 'message_id': h.gen_message_id(),
- 'destinations': [unicode(user._id)],
- 'subject': u'Bulk export for project %s completed' % project.shortname,
- 'text': tmpl.render(tmpl_context),
- }
- mail_tasks.sendmail.post(**email)
-
-
-def create_export_dir(project, export_filename):
- """Create temporary directory for export files"""
- # Name temporary directory after project shortname,
- # thus zipdir() will use proper prefix inside the archive.
- tmp_dir_suffix = os.path.splitext(export_filename)[0]
- path = os.path.join(project.bulk_export_path(), tmp_dir_suffix)
- if not os.path.exists(path):
- os.makedirs(path)
- return path
-
-
-def zip_and_cleanup(export_path, export_filename):
- """
- Zip exported data for a given path and filename.
- Copy it to proper location. Remove temporary files.
- """
- zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename))
-
- # cleanup
- shutil.rmtree(export_path)
+ except Exception as e:
+ log.error('Error exporting: %s on %s', tool, app.project.shortname, exc_info=True)
+ return None
+ else:
+ return app
+
+ def filter_successful(self, results):
+ return [result for result in results if result is not None]
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d2baf30f/Allura/allura/tests/test_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py
index 13c6f7f..d908252 100644
--- a/Allura/allura/tests/test_tasks.py
+++ b/Allura/allura/tests/test_tasks.py
@@ -447,33 +447,29 @@ class TestExportTasks(unittest.TestCase):
project = M.Project.query.get(shortname='test')
shutil.rmtree(project.bulk_export_path(), ignore_errors=True)
- @mock.patch('allura.tasks.export_tasks.log')
- def test_bulk_export_invalid_tool(self, log):
- export_tasks.bulk_export([u'bugs', u'blog'])
- log.info.assert_any_call('Can not load app for bugs mount point. Skipping.')
- log.info.assert_any_call('Can not load app for blog mount point. Skipping.')
-
- @mock.patch('allura.tasks.export_tasks.log')
- @mock.patch('allura.tasks.export_tasks.M.Project.app_instance')
- @mock.patch('allura.tasks.export_tasks.mail_tasks')
- @td.with_tool('test', 'Tickets', 'bugs')
- @td.with_tool('test', 'Blog', 'blog')
- def test_bulk_export_not_exportable_tool(self, mail_tasks, app, log):
- app.return_value.exportable = False
- export_tasks.bulk_export([u'bugs', u'blog'])
- log.info.assert_any_call('Tool bugs is not exportable. Skipping.')
- log.info.assert_any_call('Tool blog is not exportable. Skipping.')
+ def test_bulk_export_filter_exportable(self):
+ exportable = mock.Mock(exportable=True)
+ not_exportable = mock.Mock(exportable=False)
+ BE = export_tasks.BulkExport()
+ self.assertEqual(BE.filter_exportable([None, exportable, not_exportable]), [exportable])
+
+ def test_bulk_export_filter_successful(self):
+ BE = export_tasks.BulkExport()
+ self.assertEqual(BE.filter_successful(['foo', None, '0']), ['foo', '0'])
+
+ def test_get_export_path(self):
+ BE = export_tasks.BulkExport()
+ path = BE.get_export_path('/tmp/bulk_export/p/test/', 'test-0.zip')
+ self.assertEqual(path, '/tmp/bulk_export/p/test/test-0')
@mock.patch('allura.model.project.Project.__json__')
@mock.patch('allura.tasks.export_tasks.shutil')
@mock.patch('allura.tasks.export_tasks.zipdir')
@mock.patch('forgewiki.wiki_main.ForgeWikiApp.bulk_export')
- @mock.patch('allura.tasks.export_tasks.log')
@td.with_wiki
- def test_bulk_export(self, log, wiki_bulk_export, zipdir, shutil, project_json):
+ def test_bulk_export(self, wiki_bulk_export, zipdir, shutil, project_json):
M.MonQTask.query.remove()
export_tasks.bulk_export([u'wiki'])
- log.info.assert_any_call('Exporting wiki...')
wiki_bulk_export.assert_called_once()
project_json.assert_called_once()
temp = '/tmp/bulk_export/p/test/test'
@@ -493,23 +489,6 @@ class TestExportTasks(unittest.TestCase):
assert_in('The following tools were exported:\n- wiki', text)
assert_in('Sample instructions for test', text)
- def test_create_export_dir(self):
- project = M.Project.query.get(shortname='test')
- export_path = project.bulk_export_path()
- export_filename = project.bulk_export_filename()
- path = export_tasks.create_export_dir(project, export_filename)
- assert_equal(path, '/tmp/bulk_export/p/test/test')
- assert os.path.exists(os.path.join(export_path, project.shortname))
-
- @onlyif(os.path.exists(tg.config.get('scm.repos.tarball.zip_binary', '/usr/bin/zip')), 'zip binary is missing')
- def test_zip_and_cleanup(self):
- project = M.Project.query.get(shortname='test')
- export_filename = project.bulk_export_filename()
- path = export_tasks.create_export_dir(project, export_filename)
- export_tasks.zip_and_cleanup(path, export_filename)
- assert not os.path.exists(path)
- assert os.path.exists(os.path.join(project.bulk_export_path(), 'test.zip'))
-
def test_bulk_export_status(self):
assert_equal(c.project.bulk_export_status(), None)
export_tasks.bulk_export.post(['wiki'])