You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/09/19 18:21:39 UTC
[07/41] git commit: [#6638] Added importer for exported Allura JSON
[#6638] Added importer for exported Allura JSON
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/bb421077
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/bb421077
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/bb421077
Branch: refs/heads/db/5822
Commit: bb42107768cdb5dadf031f0f6a2677dae947155e
Parents: 66cb620
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Mon Sep 16 19:54:55 2013 +0000
Committer: Tim Van Steenburgh <tv...@gmail.com>
Committed: Wed Sep 18 16:13:37 2013 +0000
----------------------------------------------------------------------
Allura/allura/model/discuss.py | 7 +-
Allura/development.ini | 2 +
ForgeImporters/forgeimporters/base.py | 50 +++
ForgeImporters/forgeimporters/forge/__init__.py | 17 +
.../forge/templates/tracker/index.html | 29 ++
ForgeImporters/forgeimporters/forge/tracker.py | 208 ++++++++++++
.../forgeimporters/google/__init__.py | 19 +-
.../forgeimporters/tests/forge/__init__.py | 17 +
.../forgeimporters/tests/forge/test_tracker.py | 331 +++++++++++++++++++
.../tests/google/test_extractor.py | 4 +-
.../forgeimporters/tests/test_base.py | 44 ++-
ForgeImporters/setup.py | 1 +
ForgeTracker/forgetracker/tracker_main.py | 6 +
13 files changed, 711 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/Allura/allura/model/discuss.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/discuss.py b/Allura/allura/model/discuss.py
index 03c991e..6df982f 100644
--- a/Allura/allura/model/discuss.py
+++ b/Allura/allura/model/discuss.py
@@ -167,11 +167,14 @@ class Thread(Artifact, ActivityObject):
posts=[dict(slug=p.slug,
text=p.text,
subject=p.subject,
+ author=p.author().username,
+ timestamp=p.timestamp,
attachments=[dict(bytes=attach.length,
url=h.absurl(attach.url())) for attach in p.attachments])
for p in self.post_class().query.find(
- dict(discussion_id=self.discussion_id, thread_id=self._id, status='ok')
- )]
+ dict(discussion_id=self.discussion_id, thread_id=self._id, status='ok')
+ )
+ ]
)
@property
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 9cb4b16..2a0b542 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -147,6 +147,8 @@ bulk_export_path = /tmp/bulk_export/{nbhd}/{project}
bulk_export_filename = {project}-backup-{date:%Y-%m-%d-%H%M%S}.zip
bulk_export_download_instructions = Sample instructions for {project}
+importer_upload_path = /tmp/importer_upload/{nbhd}/{project}
+
# space-separated list of tool names that are valid options
# for project admins to set for their 'support_page' field
# this field is not used by default in Allura, so this option
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 653a172..ea01968 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -15,11 +15,18 @@
# specific language governing permissions and limitations
# under the License.
+import os
+import errno
import logging
import urllib
import urllib2
from collections import defaultdict
import traceback
+from urlparse import urlparse, parse_qs
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
from BeautifulSoup import BeautifulSoup
from tg import expose, validate, flash, redirect, config
@@ -461,3 +468,46 @@ class ImportAdminExtension(AdminExtension):
base_url = c.project.url() + 'admin/ext/'
link = SitemapEntry('Import', base_url+'import/')
sidebar_links.append(link)
+
+
+def stringio_parser(page):
+ return {
+ 'content-type': page.info()['content-type'],
+ 'data': StringIO(page.read()),
+ }
+
+class File(object):
+ def __init__(self, url, filename=None):
+ extractor = ProjectExtractor(None, url, parser=stringio_parser)
+ self.url = url
+ self.filename = filename or os.path.basename(urlparse(url).path)
+ self.type = extractor.page['content-type'].split(';')[0]
+ self.file = extractor.page['data']
+
+
+def get_importer_upload_path(project):
+ shortname = project.shortname
+ if project.is_nbhd_project:
+ shortname = project.url().strip('/')
+ elif project.is_user_project:
+ shortname = project.shortname.split('/')[1]
+ elif not project.is_root:
+ shortname = project.shortname.split('/')[0]
+ upload_path = config['importer_upload_path'].format(
+ nbhd=project.neighborhood.url_prefix.strip('/'),
+ project=shortname,
+ c=c,
+ )
+ return upload_path
+
+def save_importer_upload(project, filename, data):
+ dest_path = get_importer_upload_path(project)
+ dest_file = os.path.join(dest_path, filename)
+ try:
+ os.makedirs(dest_path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ with open(dest_file, 'w') as fp:
+ fp.write(data)
+ return dest_file
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/forge/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/forge/__init__.py b/ForgeImporters/forgeimporters/forge/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/forge/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/forge/templates/tracker/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/forge/templates/tracker/index.html b/ForgeImporters/forgeimporters/forge/templates/tracker/index.html
new file mode 100644
index 0000000..6e3a1d0
--- /dev/null
+++ b/ForgeImporters/forgeimporters/forge/templates/tracker/index.html
@@ -0,0 +1,29 @@
+{#-
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
+{% extends 'forgeimporters:templates/importer_base.html' %}
+
+{% block importer_fields %}
+ <div>
+ <label for="json">Tickets JSON
+ <br><small>JSON file exported from Allura</small>
+ </label>
+ <input name="tickets_json" type="file" />
+ {{ error('json') }}
+ </div>
+{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/forge/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/forge/tracker.py b/ForgeImporters/forgeimporters/forge/tracker.py
new file mode 100644
index 0000000..07efb0e
--- /dev/null
+++ b/ForgeImporters/forgeimporters/forge/tracker.py
@@ -0,0 +1,208 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from collections import defaultdict
+from datetime import datetime
+import json
+
+import dateutil.parser
+from formencode import validators as fev
+from pylons import tmpl_context as c
+from pylons import app_globals as g
+from ming.orm import session, ThreadLocalORMSession
+
+from tg import (
+ expose,
+ flash,
+ redirect,
+ validate,
+ )
+from tg.decorators import (
+ with_trailing_slash,
+ without_trailing_slash,
+ )
+
+from allura.controllers import BaseController
+from allura.lib import helpers as h
+from allura.lib.plugin import ImportIdConverter
+from allura.lib.decorators import require_post, task
+from allura.lib import validators as v
+from allura import model as M
+
+from forgetracker.tracker_main import ForgeTrackerApp
+from forgetracker import model as TM
+from forgeimporters.base import (
+ ToolImporter,
+ ToolImportForm,
+ ImportErrorHandler,
+ File,
+ get_importer_upload_path,
+ save_importer_upload,
+ )
+
+
+@task(notifications_disabled=True)
+def import_tool(**kw):
+ importer = ForgeTrackerImporter()
+ with ImportErrorHandler(importer, kw.get('project_name'), c.project):
+ importer.import_tool(c.project, c.user, **kw)
+
+
+class ForgeTrackerImportForm(ToolImportForm):
+ tickets_json = v.JsonFile(required=True)
+
+
+class ForgeTrackerImportController(BaseController):
+ def __init__(self):
+ self.importer = ForgeTrackerImporter()
+
+ @property
+ def target_app(self):
+ return self.importer.target_app
+
+ @with_trailing_slash
+ @expose('jinja:forgeimporters.forge:templates/tracker/index.html')
+ def index(self, **kw):
+ return dict(importer=self.importer,
+ target_app=self.target_app)
+
+ @without_trailing_slash
+ @expose()
+ @require_post()
+ @validate(ForgeTrackerImportForm(ForgeTrackerApp), error_handler=index)
+ def create(self, tickets_json, mount_point, mount_label, **kw):
+ if ForgeTrackerImporter().enforce_limit(c.project):
+ save_importer_upload(c.project, 'tickets.json', json.dumps(tickets_json))
+ import_tool.post(
+ mount_point=mount_point,
+ mount_label=mount_label,
+ )
+ flash('Ticket import has begun. Your new tracker will be available '
+ 'when the import is complete.')
+ redirect(c.project.url() + 'admin/')
+ else:
+ flash('There are too many imports pending at this time. Please wait and try again.', 'error')
+ redirect(c.project.url() + 'admin/')
+
+
+class ForgeTrackerImporter(ToolImporter):
+ source = 'Allura'
+ target_app = ForgeTrackerApp
+ controller = ForgeTrackerImportController
+ tool_label = 'Tickets'
+
+ def __init__(self, *args, **kwargs):
+ super(ForgeTrackerImporter, self).__init__(*args, **kwargs)
+ self.max_ticket_num = 0
+
+ def _load_json(self, project):
+ upload_path = get_importer_upload_path(project)
+ full_path = os.path.join(upload_path, 'tickets.json')
+ with open(full_path) as fp:
+ return json.load(fp)
+
+ def import_tool(self, project, user, mount_point=None,
+ mount_label=None, **kw):
+ import_id_converter = ImportIdConverter.get()
+ tracker_json = self._load_json(project)
+ tracker_json['tracker_config']['options'].pop('ordinal', None)
+ tracker_json['tracker_config']['options'].pop('mount_point', None)
+ tracker_json['tracker_config']['options'].pop('mount_label', None)
+ app = project.install_app('tickets', mount_point, mount_label,
+ import_id={
+ 'source': self.source,
+ 'app_config_id': tracker_json['tracker_config']['_id'],
+ },
+ open_status_names=tracker_json['open_status_names'],
+ closed_status_names=tracker_json['closed_status_names'],
+ **tracker_json['tracker_config']['options']
+ )
+ ThreadLocalORMSession.flush_all()
+ try:
+ M.session.artifact_orm_session._get().skip_mod_date = True
+ for ticket_json in tracker_json['tickets']:
+ reporter = self.get_user(ticket_json['reported_by'])
+ with h.push_config(c, user=reporter, app=app):
+ self.max_ticket_num = max(ticket_json['ticket_num'], self.max_ticket_num)
+ ticket = TM.Ticket(
+ app_config_id=app.config._id,
+ import_id=import_id_converter.expand(ticket_json['ticket_num'], app),
+ description=self.annotate(ticket_json['description'], reporter, ticket_json['reported_by']),
+ created_date=dateutil.parser.parse(ticket_json['created_date']),
+ mod_date=dateutil.parser.parse(ticket_json['mod_date']),
+ ticket_num=ticket_json['ticket_num'],
+ summary=ticket_json['summary'],
+ custom_fields=ticket_json['custom_fields'],
+ status=ticket_json['status'],
+ labels=ticket_json['labels'],
+ votes_down=ticket_json['votes_down'],
+ votes_up=ticket_json['votes_up'],
+ votes = ticket_json['votes_up'] - ticket_json['votes_down'],
+ )
+ ticket.private = ticket_json['private'] # trigger the private property
+ self.process_comments(ticket, ticket_json['discussion_thread']['posts'])
+ session(ticket).flush(ticket)
+ session(ticket).expunge(ticket)
+ app.globals.custom_fields = tracker_json['custom_fields']
+ self.process_bins(app, tracker_json['saved_bins'])
+ app.globals.last_ticket_num = self.max_ticket_num
+ M.AuditLog.log(
+ 'import tool %s from exported Allura JSON' % (
+ app.config.options.mount_point,
+ ),
+ project=project,
+ user=user,
+ url=app.url,
+ )
+ g.post_event('project_updated')
+ app.globals.invalidate_bin_counts()
+ ThreadLocalORMSession.flush_all()
+ return app
+ except Exception as e:
+ h.make_app_admin_only(app)
+ raise
+ finally:
+ M.session.artifact_orm_session._get().skip_mod_date = False
+
+ def get_user(self, username):
+ user = M.User.by_username(username)
+ if not user:
+ user = M.User.anonymous()
+ return user
+
+ def annotate(self, text, user, username):
+ if user._id is None:
+ return '*Originally by:* %s\n\n%s' % (username, text)
+ return text
+
+ def process_comments(self, ticket, comments):
+ for comment_json in comments:
+ user = self.get_user(comment_json['author'])
+ with h.push_config(c, user=user):
+ p = ticket.discussion_thread.add_post(
+ text = self.annotate(comment_json['text'], user, comment_json['author']),
+ ignore_security = True,
+ timestamp = dateutil.parser.parse(comment_json['timestamp']),
+ )
+ p.add_multiple_attachments([File(a['url']) for a in comment_json['attachments']])
+
+ def process_bins(self, app, bins):
+ TM.Bin.query.remove({'app_config_id': app.config._id})
+ for bin_json in bins:
+ bin_json.pop('_id', None)
+ TM.Bin(app_config_id=app.config._id, **bin_json)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 10ddc86..d4eb83d 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -20,10 +20,6 @@ import urllib
from urlparse import urlparse, urljoin, parse_qs
from collections import defaultdict
from contextlib import closing
-try:
- from cStringIO import StringIO
-except ImportError:
- from StringIO import StringIO
import logging
from BeautifulSoup import BeautifulSoup
@@ -31,6 +27,7 @@ from BeautifulSoup import BeautifulSoup
from allura.lib import helpers as h
from allura import model as M
from forgeimporters.base import ProjectExtractor
+from forgeimporters.base import File
log = logging.getLogger(__name__)
@@ -63,12 +60,6 @@ def csv_parser(page):
# remove CSV wrapping (quotes, commas, newlines)
return [line.strip('",\n') for line in lines]
-def stringio_parser(page):
- return {
- 'content-type': page.info()['content-type'],
- 'data': StringIO(page.read()),
- }
-
class GoogleCodeProjectExtractor(ProjectExtractor):
BASE_URL = 'http://code.google.com'
@@ -268,14 +259,6 @@ class Comment(object):
)
return text
-class File(object):
- def __init__(self, url, filename):
- extractor = GoogleCodeProjectExtractor(None, url, parser=stringio_parser)
- self.url = url
- self.filename = filename
- self.type = extractor.page['content-type'].split(';')[0]
- self.file = extractor.page['data']
-
class Attachment(File):
def __init__(self, url):
url = urljoin(GoogleCodeProjectExtractor.BASE_URL, url)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/tests/forge/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/forge/__init__.py b/ForgeImporters/forgeimporters/tests/forge/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/forge/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/forge/test_tracker.py b/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
new file mode 100644
index 0000000..9eeaec4
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/forge/test_tracker.py
@@ -0,0 +1,331 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from datetime import datetime
+from unittest import TestCase
+
+import mock
+from ming.odm import ThreadLocalORMSession
+import webtest
+
+from allura.tests import TestController
+from allura.tests.decorators import with_tracker
+
+from allura import model as M
+from forgeimporters.forge import tracker
+
+
+class TestTrackerImporter(TestCase):
+ @mock.patch.object(tracker.h, 'make_app_admin_only')
+ @mock.patch.object(tracker, 'g')
+ @mock.patch.object(tracker, 'c')
+ @mock.patch.object(tracker, 'ThreadLocalORMSession')
+ @mock.patch.object(tracker, 'session')
+ @mock.patch.object(tracker, 'M')
+ @mock.patch.object(tracker, 'TM')
+ def test_import_tool(self, TM, M, session, tlos, c, g, mao):
+ importer = tracker.ForgeTrackerImporter()
+ importer._load_json = mock.Mock(return_value={
+ 'tracker_config': {
+ '_id': 'orig_id',
+ 'options': {
+ 'foo': 'bar',
+ },
+ },
+ 'open_status_names': 'open statuses',
+ 'closed_status_names': 'closed statuses',
+ 'custom_fields': 'fields',
+ 'saved_bins': 'bins',
+ 'tickets': [
+ {
+ 'reported_by': 'rb1',
+ 'ticket_num': 1,
+ 'description': 'd1',
+ 'created_date': '2013-09-01',
+ 'mod_date': '2013-09-02',
+ 'summary': 's1',
+ 'custom_fields': 'cf1',
+ 'status': 'st1',
+ 'labels': 'l1',
+ 'votes_down': 1,
+ 'votes_up': 2,
+ 'private': False,
+ 'discussion_thread': {'posts': 'comments1'},
+ },
+ {
+ 'reported_by': 'rb2',
+ 'ticket_num': 100,
+ 'description': 'd2',
+ 'created_date': '2013-09-03',
+ 'mod_date': '2013-09-04',
+ 'summary': 's2',
+ 'custom_fields': 'cf2',
+ 'status': 'st2',
+ 'labels': 'l2',
+ 'votes_down': 3,
+ 'votes_up': 5,
+ 'private': True,
+ 'discussion_thread': {'posts': 'comments2'},
+ },
+ ],
+ })
+ reporter = mock.Mock()
+ importer.get_user = mock.Mock(return_value=reporter)
+ importer.annotate = mock.Mock(side_effect=['ad1', 'ad2'])
+ importer.process_comments = mock.Mock()
+ importer.process_bins = mock.Mock()
+ project, user = mock.Mock(), mock.Mock()
+ app = project.install_app.return_value
+ app.config.options.mount_point = 'mount_point'
+ app.config.options.import_id = {
+ 'source': 'Allura',
+ 'app_config_id': 'orig_id',
+ }
+ app.config.options.get = lambda *a: getattr(app.config.options, *a)
+ app.url = 'foo'
+ tickets = TM.Ticket.side_effect = [mock.Mock(), mock.Mock()]
+
+ importer.import_tool(project, user,
+ mount_point='mount_point', mount_label='mount_label')
+
+ project.install_app.assert_called_once_with('tickets', 'mount_point', 'mount_label',
+ open_status_names='open statuses',
+ closed_status_names='closed statuses',
+ import_id={
+ 'source': 'Allura',
+ 'app_config_id': 'orig_id',
+ },
+ foo='bar',
+ )
+ self.assertEqual(importer.annotate.call_args_list, [
+ mock.call('d1', reporter, 'rb1'),
+ mock.call('d2', reporter, 'rb2'),
+ ])
+ self.assertEqual(TM.Ticket.call_args_list, [
+ mock.call(
+ app_config_id=app.config._id,
+ import_id={
+ 'source': 'Allura',
+ 'app_config_id': 'orig_id',
+ 'source_id': 1,
+ },
+ description='ad1',
+ created_date=datetime(2013, 9, 1),
+ mod_date=datetime(2013, 9, 2),
+ ticket_num=1,
+ summary='s1',
+ custom_fields='cf1',
+ status='st1',
+ labels='l1',
+ votes_down=1,
+ votes_up=2,
+ votes=1,
+ ),
+ mock.call(
+ app_config_id=app.config._id,
+ import_id={
+ 'source': 'Allura',
+ 'app_config_id': 'orig_id',
+ 'source_id': 100,
+ },
+ description='ad2',
+ created_date=datetime(2013, 9, 3),
+ mod_date=datetime(2013, 9, 4),
+ ticket_num=100,
+ summary='s2',
+ custom_fields='cf2',
+ status='st2',
+ labels='l2',
+ votes_down=3,
+ votes_up=5,
+ votes=2,
+ ),
+ ])
+ self.assertEqual(tickets[0].private, False)
+ self.assertEqual(tickets[1].private, True)
+ self.assertEqual(importer.process_comments.call_args_list, [
+ mock.call(tickets[0], 'comments1'),
+ mock.call(tickets[1], 'comments2'),
+ ])
+ self.assertEqual(tlos.flush_all.call_args_list, [
+ mock.call(),
+ mock.call(),
+ ])
+ self.assertEqual(session.return_value.flush.call_args_list, [
+ mock.call(tickets[0]),
+ mock.call(tickets[1]),
+ ])
+ self.assertEqual(session.return_value.expunge.call_args_list, [
+ mock.call(tickets[0]),
+ mock.call(tickets[1]),
+ ])
+ self.assertEqual(app.globals.custom_fields, 'fields')
+ importer.process_bins.assert_called_once_with(app, 'bins')
+ self.assertEqual(app.globals.last_ticket_num, 100)
+ M.AuditLog.log.assert_called_once_with(
+ 'import tool mount_point from exported Allura JSON',
+ project=project, user=user, url='foo')
+ g.post_event.assert_called_once_with('project_updated')
+ app.globals.invalidate_bin_counts.assert_called_once_with()
+
+ @mock.patch.object(tracker, 'ThreadLocalORMSession')
+ @mock.patch.object(tracker, 'M')
+ @mock.patch.object(tracker, 'h')
+ def test_import_tool_failure(self, h, M, ThreadLocalORMSession):
+ M.session.artifact_orm_session._get.side_effect = ValueError
+ project = mock.Mock()
+ user = mock.Mock()
+ tracker_json = {
+ 'tracker_config': {'_id': 'orig_id', 'options': {}},
+ 'open_status_names': 'os',
+ 'closed_status_names': 'cs',
+ }
+
+ importer = tracker.ForgeTrackerImporter()
+ importer._load_json = mock.Mock(return_value=tracker_json)
+ self.assertRaises(ValueError, importer.import_tool, project, user, project_name='project_name',
+ mount_point='mount_point', mount_label='mount_label')
+
+ h.make_app_admin_only.assert_called_once_with(project.install_app.return_value)
+
+ @mock.patch.object(tracker, 'M')
+ def test_get_user(self, M):
+ importer = tracker.ForgeTrackerImporter()
+ M.User.anonymous.return_value = 'anon'
+
+ M.User.by_username.return_value = 'bar'
+ self.assertEqual(importer.get_user('foo'), 'bar')
+ self.assertEqual(M.User.anonymous.call_count, 0)
+
+ M.User.by_username.return_value = None
+ self.assertEqual(importer.get_user('foo'), 'anon')
+ self.assertEqual(M.User.anonymous.call_count, 1)
+
+ def test_annotate(self):
+ importer = tracker.ForgeTrackerImporter()
+ user = mock.Mock(_id=1)
+ self.assertEqual(importer.annotate('foo', user, 'bar'), 'foo')
+ user._id = None
+ self.assertEqual(importer.annotate('foo', user, 'bar'), '*Originally by:* bar\n\nfoo')
+
+ @mock.patch.object(tracker, 'File')
+ @mock.patch.object(tracker, 'c')
+ def test_process_comments(self, c, File):
+ importer = tracker.ForgeTrackerImporter()
+ author = mock.Mock()
+ importer.get_user = mock.Mock(return_value=author)
+ importer.annotate = mock.Mock(side_effect=['at1', 'at2'])
+ ticket = mock.Mock()
+ add_post = ticket.discussion_thread.add_post
+ ama = add_post.return_value.add_multiple_attachments
+ File.side_effect = ['f1', 'f2', 'f3', 'f4']
+ comments = [
+ {
+ 'author': 'a1',
+ 'text': 't1',
+ 'timestamp': '2013-09-01',
+ 'attachments': [{'url': 'u1'}, {'url': 'u2'}],
+ },
+ {
+ 'author': 'a2',
+ 'text': 't2',
+ 'timestamp': '2013-09-02',
+ 'attachments': [{'url': 'u3'}, {'url': 'u4'}],
+ },
+ ]
+
+ importer.process_comments(ticket, comments)
+
+ self.assertEqual(importer.get_user.call_args_list, [mock.call('a1'), mock.call('a2')])
+ self.assertEqual(importer.annotate.call_args_list, [
+ mock.call('t1', author, 'a1'),
+ mock.call('t2', author, 'a2'),
+ ])
+ self.assertEqual(add_post.call_args_list, [
+ mock.call(text='at1', ignore_security=True, timestamp=datetime(2013, 9, 1)),
+ mock.call(text='at2', ignore_security=True, timestamp=datetime(2013, 9, 2)),
+ ])
+ self.assertEqual(File.call_args_list, [
+ mock.call('u1'),
+ mock.call('u2'),
+ mock.call('u3'),
+ mock.call('u4'),
+ ])
+ self.assertEqual(ama.call_args_list, [
+ mock.call(['f1', 'f2']),
+ mock.call(['f3', 'f4']),
+ ])
+
+ @mock.patch.object(tracker, 'TM')
+ def test_process_bins(self, TM):
+ app = mock.Mock()
+ app.config._id = 1
+ importer = tracker.ForgeTrackerImporter()
+ importer.process_bins(app, [{'_id': 1, 'b': 1}, {'b': 2}])
+ TM.Bin.query.remove.assert_called_once_with({'app_config_id': 1})
+ self.assertEqual(TM.Bin.call_args_list, [
+ mock.call(app_config_id=1, b=1),
+ mock.call(app_config_id=1, b=2),
+ ])
+
+
+class TestForgeTrackerImportController(TestController, TestCase):
+ def setUp(self):
+ """Mount Allura importer on the Tracker admin controller"""
+ super(TestForgeTrackerImportController, self).setUp()
+ from forgetracker.tracker_main import TrackerAdminController
+ TrackerAdminController._importer = tracker.ForgeTrackerImportController()
+
+ @with_tracker
+ def test_index(self):
+ r = self.app.get('/p/test/admin/bugs/_importer/')
+ self.assertIsNotNone(r.html.find(attrs=dict(name="tickets_json")))
+ self.assertIsNotNone(r.html.find(attrs=dict(name="mount_label")))
+ self.assertIsNotNone(r.html.find(attrs=dict(name="mount_point")))
+
+ @with_tracker
+ @mock.patch('forgeimporters.forge.tracker.save_importer_upload')
+ @mock.patch('forgeimporters.forge.tracker.import_tool')
+ def test_create(self, import_tool, sui):
+ project = M.Project.query.get(shortname='test')
+ params = {
+ 'tickets_json': webtest.Upload('tickets.json', '{"key": "val"}'),
+ 'mount_label': 'mylabel',
+ 'mount_point': 'mymount',
+ }
+ r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+ status=302)
+ self.assertEqual(r.location, 'http://localhost/p/test/admin/')
+ sui.assert_called_once_with(project, 'tickets.json', '{"key": "val"}')
+ import_tool.post.assert_called_once_with(mount_point='mymount', mount_label='mylabel')
+
+ @with_tracker
+ @mock.patch('forgeimporters.forge.tracker.save_importer_upload')
+ @mock.patch('forgeimporters.forge.tracker.import_tool')
+ def test_create_limit(self, import_tool, sui):
+ project = M.Project.query.get(shortname='test')
+ project.set_tool_data('ForgeTrackerImporter', pending=1)
+ ThreadLocalORMSession.flush_all()
+ params = {
+ 'tickets_json': webtest.Upload('tickets.json', '{"key": "val"}'),
+ 'mount_label': 'mylabel',
+ 'mount_point': 'mymount',
+ }
+ r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+ status=302).follow()
+ self.assertIn('Please wait and try again', r)
+ self.assertEqual(import_tool.post.call_count, 0)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index a6c09be..32b3049 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -211,7 +211,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
'OpSys-OSX',
])
- @mock.patch.object(google, 'StringIO')
+ @mock.patch.object(base, 'StringIO')
def test_get_issue_attachments(self, StringIO):
self.urlopen.return_value.info.return_value = {'content-type': 'text/plain; foo'}
test_issue = open(pkg_resources.resource_filename('forgeimporters', 'tests/data/google/test-issue.html')).read()
@@ -222,7 +222,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
self.assertEqual(attachments[0].url, 'http://allura-google-importer.googlecode.com/issues/attachment?aid=70000000&name=at1.txt&token=3REU1M3JUUMt0rJUg7ldcELt6LA%3A1376059941255')
self.assertEqual(attachments[0].type, 'text/plain')
- @mock.patch.object(google, 'StringIO')
+ @mock.patch.object(base, 'StringIO')
def test_iter_comments(self, StringIO):
test_issue = open(pkg_resources.resource_filename('forgeimporters', 'tests/data/google/test-issue.html')).read()
gpe = self._make_extractor(test_issue)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/forgeimporters/tests/test_base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/test_base.py b/ForgeImporters/forgeimporters/tests/test_base.py
index ded4e37..f9a373b 100644
--- a/ForgeImporters/forgeimporters/tests/test_base.py
+++ b/ForgeImporters/forgeimporters/tests/test_base.py
@@ -16,17 +16,19 @@
# under the License.
from unittest import TestCase
+import errno
from formencode import Invalid
import mock
-from tg import expose
+from tg import expose, config
from nose.tools import assert_equal, assert_raises
from webob.exc import HTTPUnauthorized
from alluratest.controller import TestController
from allura.tests import decorators as td
+from allura.lib import helpers as h
-from .. import base
+from forgeimporters import base
class TestProjectExtractor(TestCase):
@@ -280,3 +282,41 @@ class TestProjectToolsImportController(TestController):
url = import1_page.environ['PATH_INFO']
assert url.endswith('/admin/ext/import/importer1'), url
assert_equal(import1_page.body, 'test importer 1 controller webpage')
+
+
+def test_get_importer_upload_path():
+ project = mock.Mock(
+ shortname='prefix/shortname',
+ is_nbhd_project=False,
+ is_user_project=False,
+ is_root=False,
+ url=lambda: 'n_url/',
+ neighborhood=mock.Mock(url_prefix='p/'),
+ )
+ with h.push_config(config, importer_upload_path='path/{nbhd}/{project}'):
+ assert_equal(base.get_importer_upload_path(project), 'path/p/prefix')
+ project.is_nbhd_project = True
+ assert_equal(base.get_importer_upload_path(project), 'path/p/n_url')
+ project.is_nbhd_project = False
+ project.is_user_project = True
+ assert_equal(base.get_importer_upload_path(project), 'path/p/shortname')
+ project.is_user_project = False
+ project.is_root = True
+ assert_equal(base.get_importer_upload_path(project), 'path/p/prefix/shortname')
+
+@mock.patch.object(base, 'os')
+@mock.patch.object(base, 'get_importer_upload_path')
+def test_save_importer_upload(giup, os):
+ os.path.join = lambda *a: '/'.join(a)
+ giup.return_value = 'path'
+ os.makedirs.side_effect = OSError(errno.EEXIST, 'foo')
+ _open = mock.MagicMock()
+ fp = _open.return_value.__enter__.return_value
+ with mock.patch('__builtin__.open', _open):
+ base.save_importer_upload('project', 'file', 'data')
+ os.makedirs.assert_called_once_with('path')
+ _open.assert_called_once_with('path/file', 'w')
+ fp.write.assert_called_once_with('data')
+
+ os.makedirs.side_effect = OSError(errno.EACCES, 'foo')
+ assert_raises(OSError, base.save_importer_upload, 'project', 'file', 'data')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeImporters/setup.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py
index bb8148b..eaeb7df 100644
--- a/ForgeImporters/setup.py
+++ b/ForgeImporters/setup.py
@@ -43,6 +43,7 @@ setup(name='ForgeImporters',
google-code-repo = forgeimporters.google.code:GoogleRepoImporter
github-repo = forgeimporters.github.code:GitHubRepoImporter
trac-tickets = forgeimporters.trac.tickets:TracTicketImporter
+ forge-tracker = forgeimporters.forge.tracker:ForgeTrackerImporter
[allura.admin]
importers = forgeimporters.base:ImportAdminExtension
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/bb421077/ForgeTracker/forgetracker/tracker_main.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index 6c6c20d..ea075af 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -422,6 +422,12 @@ class ForgeTrackerApp(Application):
f.write(',\n"milestones":')
milestones = self.milestones
json.dump(milestones, f, cls=jsonify.GenericJSON, indent=2)
+ f.write(',\n"custom_fields":')
+ json.dump(self.globals.custom_fields, f, cls=jsonify.GenericJSON, indent=2)
+ f.write(',\n"open_status_names":')
+ json.dump(self.globals.open_status_names, f, cls=jsonify.GenericJSON, indent=2)
+ f.write(',\n"closed_status_names":')
+ json.dump(self.globals.closed_status_names, f, cls=jsonify.GenericJSON, indent=2)
f.write(',\n"saved_bins":')
bins = self.bins
json.dump(bins, f, cls=jsonify.GenericJSON, indent=2)