You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by he...@apache.org on 2016/03/08 22:35:11 UTC
[2/9] allura git commit: [#8051] ticket:894 finalize attachments
export for wiki
[#8051] ticket:894 finalize attachments export for wiki
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/f65d6b24
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/f65d6b24
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/f65d6b24
Branch: refs/heads/master
Commit: f65d6b243f6ebe6a0564c9742630a1e472578782
Parents: 14c222a
Author: Denis Kotov <de...@gmail.com>
Authored: Thu Jan 28 16:36:13 2016 +0200
Committer: Heith Seewald <he...@gmail.com>
Committed: Tue Mar 8 14:11:56 2016 -0500
----------------------------------------------------------------------
Allura/allura/app.py | 5 ++-
Allura/allura/ext/admin/admin_main.py | 2 +-
Allura/allura/lib/utils.py | 5 +--
Allura/allura/model/discuss.py | 51 ++++++++++------------
Allura/allura/tasks/export_tasks.py | 16 +++----
ForgeBlog/forgeblog/main.py | 2 +-
ForgeDiscussion/forgediscussion/forum_main.py | 2 +-
ForgeLink/forgelink/link_main.py | 2 +-
ForgeTracker/forgetracker/tracker_main.py | 2 +-
ForgeWiki/forgewiki/model/wiki.py | 22 +++++-----
ForgeWiki/forgewiki/wiki_main.py | 16 +++----
11 files changed, 57 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/Allura/allura/app.py
----------------------------------------------------------------------
diff --git a/Allura/allura/app.py b/Allura/allura/app.py
index bd69174..f32637b 100644
--- a/Allura/allura/app.py
+++ b/Allura/allura/app.py
@@ -723,7 +723,7 @@ class Application(object):
text=text,
subject=message['headers'].get('Subject', 'no subject'))
- def bulk_export(self, f):
+ def bulk_export(self, f, export_path='', with_attachments=False):
"""Export all artifacts in the tool into json file.
:param f: File Object to write to
@@ -758,6 +758,9 @@ class Application(object):
'mount_label': self.config.options.mount_label
}
+ def get_attachemnt_path(self, path='', *args):
+ return os.path.join(path, self.config.options.mount_point, *args)
+
class AdminControllerMixin(object):
"""Provides common functionality admin controllers need"""
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/Allura/allura/ext/admin/admin_main.py
----------------------------------------------------------------------
diff --git a/Allura/allura/ext/admin/admin_main.py b/Allura/allura/ext/admin/admin_main.py
index c406e97..1cafebe 100644
--- a/Allura/allura/ext/admin/admin_main.py
+++ b/Allura/allura/ext/admin/admin_main.py
@@ -174,7 +174,7 @@ class AdminApp(Application):
def install(self, project):
pass
- def bulk_export(self, f):
+ def bulk_export(self, f, export_path='', with_attachments=False):
json.dump(self.project, f, cls=jsonify.GenericJSON, indent=2)
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/Allura/allura/lib/utils.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index e6a7341..71eeec0 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -670,9 +670,8 @@ def is_ajax(request):
return False
-class GenericJSON(json.JSONEncoder):
+class JSONForExport(tg.jsonify.GenericJSON):
def default(self, obj):
if hasattr(obj, '__json__') and callable(obj.__json__):
return obj.__json__(is_export=True)
- elif isinstance(obj, (datetime.date, datetime.datetime)):
- return str(obj)
+ return super(JSONForExport, self).default(obj)
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/Allura/allura/model/discuss.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/discuss.py b/Allura/allura/model/discuss.py
index 05c6519..24aea18 100644
--- a/Allura/allura/model/discuss.py
+++ b/Allura/allura/model/discuss.py
@@ -174,43 +174,36 @@ class Thread(Artifact, ActivityObject):
new_doc.pop('num_views', None)
return old_doc != new_doc
- def __json__(self, limit=None, page=None, is_export=False):
- if is_export:
- posts = [dict(slug=p.slug,
- text=p.text,
- subject=p.subject,
- author=p.author().username,
- timestamp=p.timestamp,
- last_edited=p.last_edit_date,
- attachments=[dict(bytes=attach.length,
- path=os.path.join(
- self.artifact.app_config.options.mount_point,
- self.artifact.title,
- self._id,
- p._id,
- attach.filename)
- ) for attach in p.attachments])
- for p in self.query_posts(status='ok', style='chronological', limit=limit, page=page)
- ]
+ def attachment_for_export(self, page):
+ return [dict(bytes=attach.length,
+ path=os.path.join(
+ self.artifact.app_config.options.mount_point,
+ self.artifact.title,
+ self._id,
+ page.slug,
+ attach.filename)
+ ) for attach in page.attachments]
+
+ def attachemtns_for_json(self, page):
+ return [dict(bytes=attach.length,
+ url=h.absurl(attach.url())) for attach in page.attachments]
- else:
- posts = [dict(slug=p.slug,
+ def __json__(self, limit=None, page=None, is_export=False):
+ return dict(
+ _id=self._id,
+ discussion_id=str(self.discussion_id),
+ subject=self.subject,
+ limit=limit,
+ page=page,
+ posts=[dict(slug=p.slug,
text=p.text,
subject=p.subject,
author=p.author().username,
timestamp=p.timestamp,
last_edited=p.last_edit_date,
- attachments=[dict(bytes=attach.length,
- url=h.absurl(attach.url())) for attach in p.attachments])
+ attachments=self.attachment_for_export(p) if is_export else self.attachemtns_for_json(p))
for p in self.query_posts(status='ok', style='chronological', limit=limit, page=page)
]
- return dict(
- _id=self._id,
- discussion_id=str(self.discussion_id),
- subject=self.subject,
- limit=limit,
- page=page,
- posts=posts
)
@property
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/Allura/allura/tasks/export_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/export_tasks.py b/Allura/allura/tasks/export_tasks.py
index 238953d..9e2d73e 100644
--- a/Allura/allura/tasks/export_tasks.py
+++ b/Allura/allura/tasks/export_tasks.py
@@ -33,7 +33,7 @@ log = logging.getLogger(__name__)
@task
-def bulk_export(tools, filename=None, send_email=True):
+def bulk_export(tools, filename=None, send_email=True, with_attachments=False):
'''
Export the current project data. Send notification to current user.
@@ -42,12 +42,12 @@ def bulk_export(tools, filename=None, send_email=True):
'''
# it's very handy to use c.* within a @task,
# but let's be explicit and keep it separate from the main code
- return BulkExport().process(c.project, tools, c.user, filename, send_email)
+ return BulkExport().process(c.project, tools, c.user, filename, send_email, with_attachments)
class BulkExport(object):
- def process(self, project, tools, user, filename=None, send_email=True):
+ def process(self, project, tools, user, filename=None, send_email=True, with_attachments=False):
export_filename = filename or project.bulk_export_filename()
export_path = self.get_export_path(
project.bulk_export_path(), export_filename)
@@ -55,7 +55,7 @@ class BulkExport(object):
os.makedirs(export_path)
apps = [project.app_instance(tool) for tool in tools]
exportable = self.filter_exportable(apps)
- results = [self.export(export_path, app) for app in exportable]
+ results = [self.export(export_path, app, with_attachments) for app in exportable]
exported = self.filter_successful(results)
if exported:
zipdir(export_path,
@@ -107,16 +107,12 @@ class BulkExport(object):
def filter_exportable(self, apps):
return [app for app in apps if app and app.exportable]
- def export(self, export_path, app):
+ def export(self, export_path, app, with_attachments=False):
tool = app.config.options.mount_point
json_file = os.path.join(export_path, '%s.json' % tool)
try:
with open(json_file, 'w') as f:
- try:
- app.bulk_export(f, export_path, True)
- except Exception, e:
- log.info(e)
- app.bulk_export(f)
+ app.bulk_export(f, export_path, with_attachments)
except Exception:
log.error('Error exporting: %s on %s', tool,
app.project.shortname, exc_info=True)
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/ForgeBlog/forgeblog/main.py
----------------------------------------------------------------------
diff --git a/ForgeBlog/forgeblog/main.py b/ForgeBlog/forgeblog/main.py
index 9154a33..98482ce 100644
--- a/ForgeBlog/forgeblog/main.py
+++ b/ForgeBlog/forgeblog/main.py
@@ -205,7 +205,7 @@ class ForgeBlogApp(Application):
BM.BlogPostSnapshot.query.remove(dict(app_config_id=c.app.config._id))
super(ForgeBlogApp, self).uninstall(project)
- def bulk_export(self, f):
+ def bulk_export(self, f, export_path='', with_attachments=False):
f.write('{"posts": [')
posts = BM.BlogPost.query.find(dict(app_config_id=self.config._id))
for i, post in enumerate(posts):
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/ForgeDiscussion/forgediscussion/forum_main.py
----------------------------------------------------------------------
diff --git a/ForgeDiscussion/forgediscussion/forum_main.py b/ForgeDiscussion/forgediscussion/forum_main.py
index 914dab8..276eb32 100644
--- a/ForgeDiscussion/forgediscussion/forum_main.py
+++ b/ForgeDiscussion/forgediscussion/forum_main.py
@@ -235,7 +235,7 @@ class ForgeDiscussionApp(Application):
DM.ForumPost.query.remove(dict(app_config_id=self.config._id))
super(ForgeDiscussionApp, self).uninstall(project)
- def bulk_export(self, f):
+ def bulk_export(self, f, export_path='', with_attachments=False):
f.write('{"forums": [')
forums = DM.Forum.query.find(dict(app_config_id=self.config._id))
for i, forum in enumerate(forums):
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/ForgeLink/forgelink/link_main.py
----------------------------------------------------------------------
diff --git a/ForgeLink/forgelink/link_main.py b/ForgeLink/forgelink/link_main.py
index 8dc69a2..4b3ceb6 100644
--- a/ForgeLink/forgelink/link_main.py
+++ b/ForgeLink/forgelink/link_main.py
@@ -107,7 +107,7 @@ class ForgeLinkApp(Application):
"Remove all the tool's artifacts from the database"
super(ForgeLinkApp, self).uninstall(project)
- def bulk_export(self, f):
+ def bulk_export(self, f, export_path='', with_attachments=False):
json.dump(RootRestController(self).link_json(),
f, cls=jsonify.GenericJSON, indent=2)
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/ForgeTracker/forgetracker/tracker_main.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index cbf66b3..bf99256 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -477,7 +477,7 @@ class ForgeTrackerApp(Application):
TM.Globals.query.remove(app_config_id)
super(ForgeTrackerApp, self).uninstall(project)
- def bulk_export(self, f):
+ def bulk_export(self, f, export_path='', with_attachments=False):
f.write('{"tickets": [')
tickets = TM.Ticket.query.find(dict(
app_config_id=self.config._id,
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/ForgeWiki/forgewiki/model/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/model/wiki.py b/ForgeWiki/forgewiki/model/wiki.py
index 68283ef..5d75136 100644
--- a/ForgeWiki/forgewiki/model/wiki.py
+++ b/ForgeWiki/forgewiki/model/wiki.py
@@ -124,21 +124,23 @@ class Page(VersionedArtifact, ActivityObject):
d.update(summary=self.title)
return d
+ def attachments_for_export(self):
+ return [dict(bytes=attach.length,
+ path=os.path.join(
+ self.app_config.options.mount_point,
+ self.title,
+ attach.filename)) for attach in self.attachments]
+
+ def attachments_for_json(self):
+ return [dict(bytes=attach.length,
+ url=h.absurl(attach.url())) for attach in self.attachments]
+
def __json__(self, posts_limit=None, is_export=False):
- if is_export:
- attachments = [dict(bytes=attach.length,
- path=os.path.join(
- self.app_config.options.mount_point,
- self.title,
- attach.filename)) for attach in self.attachments]
- else:
- attachments = [dict(bytes=attach.length,
- url=h.absurl(attach.url())) for attach in self.attachments]
return dict(super(Page, self).__json__(posts_limit=posts_limit, is_export=is_export),
title=self.title,
text=self.text,
labels=list(self.labels),
- attachments=attachments)
+ attachments=self.attachments_for_export() if is_export else self.attachments_for_json())
def commit(self):
ss = VersionedArtifact.commit(self)
http://git-wip-us.apache.org/repos/asf/allura/blob/f65d6b24/ForgeWiki/forgewiki/wiki_main.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/wiki_main.py b/ForgeWiki/forgewiki/wiki_main.py
index 275cbdf..54ca706 100644
--- a/ForgeWiki/forgewiki/wiki_main.py
+++ b/ForgeWiki/forgewiki/wiki_main.py
@@ -38,7 +38,7 @@ from allura.app import Application, SitemapEntry, DefaultAdminController, Config
from allura.lib.search import search_app
from allura.lib.decorators import require_post
from allura.lib.security import require_access, has_access
-from allura.lib.utils import is_ajax, GenericJSON
+from allura.lib.utils import is_ajax, JSONForExport
from allura.lib import exceptions as forge_exc
from allura.controllers import AppDiscussionController, BaseController, AppDiscussionRestController
from allura.controllers import DispatchIndex
@@ -345,8 +345,8 @@ The wiki uses [Markdown](%s) syntax.
app_config_id=self.config._id,
deleted=False)))
if with_attachments:
- GenericClass = GenericJSON
- self.export_attachment(pages, export_path)
+ GenericClass = JSONForExport
+ self.export_attachments(pages, export_path)
else:
GenericClass = jsonify.GenericJSON
for i, page in enumerate(pages):
@@ -355,13 +355,9 @@ The wiki uses [Markdown](%s) syntax.
json.dump(page, f, cls=GenericClass, indent=2)
f.write(']}')
- def export_attachment(self, pages, export_path):
+ def export_attachments(self, pages, export_path):
for page in pages:
- attachment_path = os.path.join(
- export_path,
- self.config.options.mount_point,
- page.title
- )
+ attachment_path = self.get_attachemnt_path(export_path, page.title)
if not os.path.exists(attachment_path):
os.makedirs(attachment_path)
for attachment in page.attachments:
@@ -373,7 +369,7 @@ The wiki uses [Markdown](%s) syntax.
post_path = os.path.join(
attachment_path,
page.discussion_thread._id,
- post._id
+ post.slug
)
if not os.path.exists(post_path):
os.makedirs(post_path)