You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2018/06/26 21:19:25 UTC
allura git commit: [#8211] support config option for having project
exports use a separate tmp dir
Repository: allura
Updated Branches:
refs/heads/db/8211 [created] 04ea81380
[#8211] support config option for having project exports use a separate tmp dir
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/04ea8138
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/04ea8138
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/04ea8138
Branch: refs/heads/db/8211
Commit: 04ea813801ac32ef11f253011b584cff8b0e8c6d
Parents: a23b41a
Author: Dave Brondsema <da...@brondsema.net>
Authored: Tue Jun 26 17:19:13 2018 -0400
Committer: Dave Brondsema <da...@brondsema.net>
Committed: Tue Jun 26 17:19:13 2018 -0400
----------------------------------------------------------------------
Allura/allura/model/project.py | 8 ++++++--
Allura/allura/tasks/export_tasks.py | 28 ++++++++++++----------------
Allura/allura/tests/test_tasks.py | 9 ++-------
Allura/development.ini | 1 +
4 files changed, 21 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/04ea8138/Allura/allura/model/project.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/project.py b/Allura/allura/model/project.py
index 7eac462..5420adb 100644
--- a/Allura/allura/model/project.py
+++ b/Allura/allura/model/project.py
@@ -1124,7 +1124,11 @@ class Project(SearchIndexable, MappedClass, ActivityNode, ActivityObject):
accounturl=accounturl
))
- def bulk_export_path(self):
+ def bulk_export_path(self, rootdir=None):
+ """
+ :param rootdir: a directory path, using {nbhd} {project} or {c..} substitution vars if desired
+ :return: a bulk export path for the current project
+ """
shortname = self.shortname
if self.is_nbhd_project:
shortname = self.url().strip('/')
@@ -1132,7 +1136,7 @@ class Project(SearchIndexable, MappedClass, ActivityNode, ActivityObject):
shortname = self.shortname.split('/')[1]
elif not self.is_root:
shortname = self.shortname.split('/')[0]
- return config['bulk_export_path'].format(
+ return rootdir.format(
nbhd=self.neighborhood.url_prefix.strip('/'),
project=shortname,
c=c,
http://git-wip-us.apache.org/repos/asf/allura/blob/04ea8138/Allura/allura/tasks/export_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/export_tasks.py b/Allura/allura/tasks/export_tasks.py
index c956148..d63c40f 100644
--- a/Allura/allura/tasks/export_tasks.py
+++ b/Allura/allura/tasks/export_tasks.py
@@ -49,18 +49,23 @@ class BulkExport(object):
def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False):
export_filename = filename or project.bulk_export_filename()
- export_path = self.get_export_path(
- project.bulk_export_path(), export_filename)
+ tmp_path = os.path.join(
+ project.bulk_export_path(rootdir=tg.config.get('bulk_export_tmpdir', tg.config['bulk_export_path'])),
+ os.path.splitext(export_filename)[0], # e.g. test-backup-2018-06-26-210524 without the .zip
+ )
+ export_path = project.bulk_export_path(rootdir=tg.config['bulk_export_path'])
+ export_fullpath = os.path.join(export_path, export_filename)
+ if not os.path.exists(tmp_path):
+ os.makedirs(tmp_path)
if not os.path.exists(export_path):
os.makedirs(export_path)
apps = [project.app_instance(tool) for tool in tools]
exportable = self.filter_exportable(apps)
- results = [self.export(export_path, app, with_attachments) for app in exportable]
+ results = [self.export(tmp_path, app, with_attachments) for app in exportable]
exported = self.filter_successful(results)
if exported:
- zipdir(export_path,
- os.path.join(os.path.dirname(export_path), export_filename))
- shutil.rmtree(export_path.encode('utf8')) # must encode into bytes or it'll fail on non-ascii filenames
+ zipdir(tmp_path, export_fullpath)
+ shutil.rmtree(tmp_path.encode('utf8')) # must encode into bytes or it'll fail on non-ascii filenames
if not user:
log.info('No user. Skipping notification.')
@@ -68,8 +73,7 @@ class BulkExport(object):
if not send_email:
return
- tmpl = g.jinja2_env.get_template(
- 'allura:templates/mail/bulk_export.html')
+ tmpl = g.jinja2_env.get_template('allura:templates/mail/bulk_export.html')
instructions = tg.config.get('bulk_export_download_instructions', '')
instructions = instructions.format(
project=project.shortname,
@@ -88,14 +92,6 @@ class BulkExport(object):
u'Bulk export for project %s completed' % project.shortname,
tmpl.render(tmpl_context))
- def get_export_path(self, export_base_path, export_filename):
- """Create temporary directory for export files"""
- # Name temporary directory after project shortname,
- # thus zipdir() will use proper prefix inside the archive.
- tmp_dir_suffix = os.path.splitext(export_filename)[0]
- path = os.path.join(export_base_path, tmp_dir_suffix)
- return path
-
def filter_exportable(self, apps):
return [app for app in apps if app and app.exportable]
http://git-wip-us.apache.org/repos/asf/allura/blob/04ea8138/Allura/allura/tests/test_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py
index 2f4de42..7aefa9c 100644
--- a/Allura/allura/tests/test_tasks.py
+++ b/Allura/allura/tests/test_tasks.py
@@ -525,11 +525,11 @@ class TestExportTasks(unittest.TestCase):
setup_basic_test()
setup_global_objects()
project = M.Project.query.get(shortname='test')
- shutil.rmtree(project.bulk_export_path(), ignore_errors=True)
+ shutil.rmtree(project.bulk_export_path(tg.config['bulk_export_path']), ignore_errors=True)
def tearDown(self):
project = M.Project.query.get(shortname='test')
- shutil.rmtree(project.bulk_export_path(), ignore_errors=True)
+ shutil.rmtree(project.bulk_export_path(tg.config['bulk_export_path']), ignore_errors=True)
def test_bulk_export_filter_exportable(self):
exportable = mock.Mock(exportable=True)
@@ -543,11 +543,6 @@ class TestExportTasks(unittest.TestCase):
self.assertEqual(
BE.filter_successful(['foo', None, '0']), ['foo', '0'])
- def test_get_export_path(self):
- BE = export_tasks.BulkExport()
- path = BE.get_export_path('/tmp/bulk_export/p/test/', 'test-0.zip')
- self.assertEqual(path, '/tmp/bulk_export/p/test/test-0')
-
@mock.patch('allura.tasks.export_tasks.shutil')
@mock.patch('allura.tasks.export_tasks.zipdir')
@mock.patch('forgewiki.wiki_main.ForgeWikiApp.bulk_export')
http://git-wip-us.apache.org/repos/asf/allura/blob/04ea8138/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 9e19edb..561c410 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -414,6 +414,7 @@ scm.view.commit_browser.limit = 500
; bulk_export_enabled = true
; If you keep bulk_export_enabled, you should set up your server to securely share bulk_export_path with users somehow
bulk_export_path = /tmp/bulk_export/{nbhd}/{project}
+; bulk_export_tmpdir can be set to hold files before building the zip file. Defaults to use bulk_export_path
bulk_export_filename = {project}-backup-{date:%Y-%m-%d-%H%M%S}.zip
; You will need to specify site-specific instructions here for accessing the exported files.
bulk_export_download_instructions = Sample instructions for {project}