You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2013/08/28 16:35:05 UTC
[24/50] git commit: [#3154] simplify zipping & cleanup
[#3154] simplify zipping & cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d91d964e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d91d964e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d91d964e
Branch: refs/heads/tv/6526
Commit: d91d964e24f8960fe7cd7e7d94198f4632f57a09
Parents: 7a5bf4c
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Thu Aug 22 20:04:25 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu Aug 22 21:28:23 2013 +0000
----------------------------------------------------------------------
Allura/allura/ext/admin/admin_main.py | 2 +-
Allura/allura/tasks/export_tasks.py | 29 +++++++++++------------------
Allura/allura/tests/test_tasks.py | 22 +++++++---------------
3 files changed, 19 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d91d964e/Allura/allura/ext/admin/admin_main.py
----------------------------------------------------------------------
diff --git a/Allura/allura/ext/admin/admin_main.py b/Allura/allura/ext/admin/admin_main.py
index 0e33c94..22bd228 100644
--- a/Allura/allura/ext/admin/admin_main.py
+++ b/Allura/allura/ext/admin/admin_main.py
@@ -653,7 +653,7 @@ class ProjectAdminController(BaseController):
flash('Export for project %s already running' % c.project.shortname, 'info')
redirect('export')
export_tasks.bulk_export.post(tools)
- flash('Export scheduled', 'ok')
+ flash('Export scheduled. You will recieve an email with download instructions when complete.', 'ok')
redirect('export')
return {
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d91d964e/Allura/allura/tasks/export_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/export_tasks.py b/Allura/allura/tasks/export_tasks.py
index 06590021..d06105e 100644
--- a/Allura/allura/tasks/export_tasks.py
+++ b/Allura/allura/tasks/export_tasks.py
@@ -16,6 +16,7 @@
# under the License.
import os
+import os.path
import logging
import shutil
from tempfile import mkstemp
@@ -47,6 +48,7 @@ def bulk_export(tools):
def _bulk_export(project, tools, user):
export_filename = project.bulk_export_filename()
+ export_path = create_export_dir(project, export_filename)
not_exported_tools = []
for tool in tools or []:
app = project.app_instance(tool)
@@ -60,12 +62,9 @@ def _bulk_export(project, tools, user):
continue
log.info('Exporting %s...' % tool)
try:
- path = create_export_dir(project, export_filename)
- temp_name = mkstemp(dir=path)[1]
- with open(temp_name, 'w') as f:
- with h.push_context(project._id):
- app.bulk_export(f)
- os.rename(temp_name, os.path.join(path, '%s.json' % tool))
+ json_file = os.path.join(export_path, '%s.json' % tool)
+ with open(json_file, 'w') as f:
+ app.bulk_export(f)
except:
log.error('Something went wrong during export of %s' % tool, exc_info=True)
not_exported_tools.append(tool)
@@ -74,7 +73,7 @@ def _bulk_export(project, tools, user):
if tools and len(not_exported_tools) < len(tools):
# If that fails, we need to let it fail
# there won't be a valid zip file for the user to get.
- zip_and_cleanup(project, export_filename)
+ zip_and_cleanup(export_path, export_filename)
else:
log.error('Nothing to export')
return None
@@ -106,25 +105,19 @@ def create_export_dir(project, export_filename):
"""Create temporary directory for export files"""
# Name temporary directory after project shortname,
# thus zipdir() will use proper prefix inside the archive.
- tmp_dir_suffix = export_filename.split('.')[0]
+ tmp_dir_suffix = os.path.splitext(export_filename)[0]
path = os.path.join(project.bulk_export_path(), tmp_dir_suffix)
if not os.path.exists(path):
os.makedirs(path)
return path
-def zip_and_cleanup(project, export_filename):
+def zip_and_cleanup(export_path, export_filename):
"""
- Zip exported data for a given project and filename (no path).
+ Zip exported data for a given path and filename.
Copy it to proper location. Remove temporary files.
"""
- path = project.bulk_export_path()
- temp = os.path.join(path, export_filename.split('.')[0])
- zip_path_temp = os.path.join(temp, export_filename)
- zip_path = os.path.join(path, export_filename)
-
- zipdir(temp, zip_path_temp)
+ zipdir(export_path, os.path.join(os.path.dirname(export_path), export_filename))
# cleanup
- shutil.move(zip_path_temp, zip_path)
- shutil.rmtree(temp)
\ No newline at end of file
+ shutil.rmtree(export_path)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d91d964e/Allura/allura/tests/test_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_tasks.py b/Allura/allura/tests/test_tasks.py
index 26bcb36..80ecdfa 100644
--- a/Allura/allura/tests/test_tasks.py
+++ b/Allura/allura/tests/test_tasks.py
@@ -344,10 +344,8 @@ class TestExportTasks(unittest.TestCase):
@mock.patch('allura.tasks.export_tasks.log')
def test_bulk_export_invalid_tool(self, log):
export_tasks.bulk_export([u'bugs', u'blog'])
- assert_equal(log.info.call_count, 2)
- assert_equal(log.info.call_args_list, [
- mock.call('Can not load app for bugs mount point. Skipping.'),
- mock.call('Can not load app for blog mount point. Skipping.')])
+ log.info.assert_any_call('Can not load app for bugs mount point. Skipping.')
+ log.info.assert_any_call('Can not load app for blog mount point. Skipping.')
@mock.patch('allura.tasks.export_tasks.log')
@mock.patch('allura.tasks.export_tasks.M.Project.app_instance')
@@ -357,10 +355,8 @@ class TestExportTasks(unittest.TestCase):
def test_bulk_export_not_exportable_tool(self, mail_tasks, app, log):
app.return_value.exportable = False
export_tasks.bulk_export([u'bugs', u'blog'])
- assert_equal(log.info.call_count, 2)
- assert_equal(log.info.call_args_list, [
- mock.call('Tool bugs is not exportable. Skipping.'),
- mock.call('Tool blog is not exportable. Skipping.')])
+ log.info.assert_any_call('Tool bugs is not exportable. Skipping.')
+ log.info.assert_any_call('Tool blog is not exportable. Skipping.')
@mock.patch('allura.model.project.Project.__json__')
@mock.patch('allura.tasks.export_tasks.shutil')
@@ -371,15 +367,12 @@ class TestExportTasks(unittest.TestCase):
def test_bulk_export(self, log, wiki_bulk_export, zipdir, shutil, project_json):
M.MonQTask.query.remove()
export_tasks.bulk_export([u'wiki'])
- assert_equal(log.info.call_count, 1)
- assert_equal(log.info.call_args_list, [
- mock.call('Exporting wiki...')])
+ log.info.assert_any_call('Exporting wiki...')
wiki_bulk_export.assert_called_once()
project_json.assert_called_once()
temp = '/tmp/bulk_export/p/test/test'
zipfn = '/tmp/bulk_export/p/test/test.zip'
zipdir.assert_caled_with(temp, temp + '/test.zip')
- shutil.move.assert_called_once_with(temp + '/test.zip', zipfn)
shutil.rmtree.assert_called_once_with(temp)
# check notification
M.MonQTask.run_ready()
@@ -403,12 +396,11 @@ class TestExportTasks(unittest.TestCase):
@onlyif(os.path.exists(tg.config.get('scm.repos.tarball.zip_binary', '/usr/bin/zip')), 'zip binary is missing')
def test_zip_and_cleanup(self):
project = M.Project.query.get(shortname='test')
- export_path = project.bulk_export_path()
export_filename = project.bulk_export_filename()
path = export_tasks.create_export_dir(project, export_filename)
- export_tasks.zip_and_cleanup(project, export_filename)
+ export_tasks.zip_and_cleanup(path, export_filename)
assert not os.path.exists(path)
- assert os.path.exists(os.path.join(export_path, 'test.zip'))
+ assert os.path.exists(os.path.join(project.bulk_export_path(), 'test.zip'))
def test_bulk_export_status(self):
assert_equal(c.project.bulk_export_status(), None)