You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by tv...@apache.org on 2013/08/20 15:28:35 UTC
[09/25] git commit: [#6480] Improvements to Trac importer
[#6480] Improvements to Trac importer
- Disable email notifications during import
- Retry time-out HTTP requests
- Add option to supply user-map file
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/675495e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/675495e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/675495e3
Branch: refs/heads/db/6482
Commit: 675495e39723e593cba00cdcf84e3cb02d00e4b0
Parents: 6657808
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Mon Aug 12 18:06:43 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Thu Aug 15 16:08:41 2013 +0000
----------------------------------------------------------------------
Allura/allura/lib/decorators.py | 75 +++++++++++---------
Allura/allura/lib/helpers.py | 71 +++++++++++++++++-
Allura/allura/lib/import_api.py | 2 +-
Allura/allura/lib/spam/__init__.py | 2 +-
Allura/allura/lib/validators.py | 30 ++++++++
Allura/allura/scripts/trac_export.py | 9 ++-
Allura/allura/tests/test_decorators.py | 39 ++++++++++
Allura/allura/tests/test_helpers.py | 42 +++++++++++
Allura/allura/tests/test_validators.py | 36 ++++++++++
ForgeBlog/forgeblog/command/rssfeeds.py | 2 +-
ForgeImporters/forgeimporters/base.py | 5 +-
.../forgeimporters/templates/project_base.html | 11 ++-
ForgeImporters/forgeimporters/trac/project.py | 2 +
.../forgeimporters/trac/templates/project.html | 14 +++-
.../trac/templates/tickets/index.html | 5 +-
.../forgeimporters/trac/tests/test_tickets.py | 12 +++-
ForgeImporters/forgeimporters/trac/tickets.py | 12 ++--
ForgeTracker/forgetracker/import_support.py | 24 +++++--
ForgeTracker/forgetracker/tracker_main.py | 23 +++---
ForgeWiki/forgewiki/wiki_main.py | 23 +++---
20 files changed, 354 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index 36fb2de..137152e 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -28,16 +28,49 @@ from tg import request, redirect
from webob import exc
-def task(func):
- '''Decorator to add some methods to task functions'''
- def post(*args, **kwargs):
- from allura import model as M
- delay = kwargs.pop('delay', 0)
- return M.MonQTask.post(func, args, kwargs, delay=delay)
- # if decorating a class, have to make it a staticmethod
- # or it gets a spurious cls argument
- func.post = staticmethod(post) if inspect.isclass(func) else post
- return func
+from pylons import tmpl_context as c
+from allura.lib import helpers as h
+
+
+def _get_model():
+ from allura import model as M
+ return M
+
+def task(*args, **kw):
+ """Decorator that adds a ``.post()`` function to the decorated callable.
+
+ Calling <original_callable>.post(*args, **kw) queues the callable for
+ execution by a background worker process. All parameters must be
+ BSON-serializable.
+
+ Example usage:
+
+ @task
+ def myfunc():
+ pass
+
+ @task(notifications_disabled=True)
+ def myotherfunc():
+ # No email notifications will be sent for c.project during this task
+ pass
+
+ """
+ def task_(func):
+ def post(*args, **kwargs):
+ delay = kwargs.pop('delay', 0)
+ project = getattr(c, 'project', None)
+ cm = (h.notifications_disabled if project and
+ kw.get('notifications_disabled') else h.null_contextmanager)
+ with cm(project):
+ M = _get_model()
+ return M.MonQTask.post(func, args, kwargs, delay=delay)
+ # if decorating a class, have to make it a staticmethod
+ # or it gets a spurious cls argument
+ func.post = staticmethod(post) if inspect.isclass(func) else post
+ return func
+ if len(args) == 1 and callable(args[0]):
+ return task_(args[0])
+ return task_
class event_handler(object):
'''Decorator to register event handlers'''
@@ -155,28 +188,6 @@ class log_action(object): # pragma no cover
extra['referer_link'] = referer_link
return extra
-class exceptionless(object):
- '''Decorator making the decorated function return 'error_result' on any
- exceptions rather than propagating exceptions up the stack
- '''
-
- def __init__(self, error_result, log=None):
- self.error_result = error_result
- self.log = log
-
- def __call__(self, fun):
- fname = 'exceptionless(%s)' % fun.__name__
- def inner(*args, **kwargs):
- try:
- return fun(*args, **kwargs)
- except Exception as e:
- if self.log:
- self.log.exception('Error calling %s(args=%s, kwargs=%s): %s',
- fname, args, kwargs, str(e))
- return self.error_result
- inner.__name__ = fname
- return inner
-
def Property(function):
'''Decorator to easily assign descriptors based on sub-function names
See <http://code.activestate.com/recipes/410698-property-decorator-for-python-24/>
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index 1604aa9..0511dc7 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -22,6 +22,7 @@ import os
import os.path
import difflib
import urllib
+import urllib2
import re
import json
import logging
@@ -30,6 +31,7 @@ from hashlib import sha1
from datetime import datetime, timedelta
from collections import defaultdict
import shlex
+import socket
import tg
import genshi.template
@@ -53,10 +55,10 @@ from webhelpers import date, feedgenerator, html, number, misc, text
from allura.lib import exceptions as exc
# Reimport to make available to templates
-from allura.lib.decorators import exceptionless
from allura.lib import AsciiDammit
from .security import has_access
+log = logging.getLogger(__name__)
# validates project, subproject, and user names
re_project_name = re.compile(r'^[a-z][-a-z0-9]{2,14}$')
@@ -857,3 +859,70 @@ def split_select_field_options(field_options):
# so we're getting rid of those.
field_options = [o.replace('"', '') for o in field_options]
return field_options
+
+
+@contextmanager
+def notifications_disabled(project):
+ """Temporarily disable email notifications on a project.
+
+ """
+ orig = project.notifications_disabled
+ try:
+ project.notifications_disabled = True
+ yield
+ finally:
+ project.notifications_disabled = orig
+
+
+@contextmanager
+def null_contextmanager(*args, **kw):
+ """A no-op contextmanager.
+
+ """
+ yield
+
+
+class exceptionless(object):
+ '''Decorator making the decorated function return 'error_result' on any
+ exceptions rather than propagating exceptions up the stack
+ '''
+
+ def __init__(self, error_result, log=None):
+ self.error_result = error_result
+ self.log = log
+
+ def __call__(self, fun):
+ fname = 'exceptionless(%s)' % fun.__name__
+ def inner(*args, **kwargs):
+ try:
+ return fun(*args, **kwargs)
+ except Exception as e:
+ if self.log:
+ self.log.exception('Error calling %s(args=%s, kwargs=%s): %s',
+ fname, args, kwargs, str(e))
+ return self.error_result
+ inner.__name__ = fname
+ return inner
+
+
+def urlopen(url, retries=3, codes=(408,)):
+ """Open url, optionally retrying if an error is encountered.
+
+ Socket timeouts will always be retried if retries > 0.
+ HTTP errors are retried if the error code is passed in ``codes``.
+
+ :param retries: Number of time to retry.
+ :param codes: HTTP error codes that should be retried.
+
+ """
+ while True:
+ try:
+ return urllib2.urlopen(url)
+ except (urllib2.HTTPError, socket.timeout) as e:
+ if retries and (isinstance(e, socket.timeout) or
+ e.code in codes):
+ retries -= 1
+ continue
+ else:
+ log.exception('Failed after %s retries: %s', retries, e)
+ raise
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/import_api.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/import_api.py b/Allura/allura/lib/import_api.py
index 1f3f98d..8eeb8a7 100644
--- a/Allura/allura/lib/import_api.py
+++ b/Allura/allura/lib/import_api.py
@@ -49,7 +49,7 @@ class AlluraImportApiClient(object):
url = urlparse.urljoin(self.base_url, url)
if self.verbose:
print "Using URL '%s'" % (url)
-
+
params = self.sign(urlparse.urlparse(url).path, params.items())
while True:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/spam/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/__init__.py b/Allura/allura/lib/spam/__init__.py
index 8e6607a..cfb5c41 100644
--- a/Allura/allura/lib/spam/__init__.py
+++ b/Allura/allura/lib/spam/__init__.py
@@ -17,7 +17,7 @@
import logging
-from allura.lib.decorators import exceptionless
+from allura.lib.helpers import exceptionless
log = logging.getLogger(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/validators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/validators.py b/Allura/allura/lib/validators.py
index ae863e8..9b44d53 100644
--- a/Allura/allura/lib/validators.py
+++ b/Allura/allura/lib/validators.py
@@ -151,6 +151,36 @@ class JsonConverter(fev.FancyValidator):
raise fe.Invalid('Invalid JSON: ' + str(e), value, state)
return obj
+class JsonFile(fev.FieldStorageUploadConverter):
+ """Validates that a file is JSON and returns the deserialized Python object
+
+ """
+ def _to_python(self, value, state):
+ return JsonConverter.to_python(value.value)
+
+class UserMapJsonFile(JsonFile):
+ """Validates that a JSON file conforms to this format:
+
+ {str:str, ...}
+
+ and returns a deserialized or stringified copy of it.
+
+ """
+ def __init__(self, as_string=False):
+ self.as_string = as_string
+
+ def _to_python(self, value, state):
+ value = super(self.__class__, self)._to_python(value, state)
+ try:
+ for k, v in value.iteritems():
+ if not(isinstance(k, basestring) and isinstance(v, basestring)):
+ raise
+ return json.dumps(value) if self.as_string else value
+ except:
+ raise fe.Invalid(
+ 'User map file must contain mapping of {str:str, ...}',
+ value, state)
+
class CreateTaskSchema(fe.Schema):
task = TaskValidator(not_empty=True, strip=True)
task_args = JsonConverter(if_missing=dict(args=[], kwargs={}))
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
index e45175b..d53afbc 100644
--- a/Allura/allura/scripts/trac_export.py
+++ b/Allura/allura/scripts/trac_export.py
@@ -18,6 +18,7 @@
# under the License.
import logging
+import socket
import sys
import csv
import urlparse
@@ -34,6 +35,8 @@ from BeautifulSoup import BeautifulSoup, NavigableString
import dateutil.parser
import pytz
+from allura.lib import helpers as h
+
log = logging.getLogger(__name__)
@@ -121,7 +124,7 @@ class TracExport(object):
def csvopen(self, url):
self.log_url(url)
- f = urllib2.urlopen(url)
+ f = h.urlopen(url)
# Trac doesn't throw 403 error, just shows normal 200 HTML page
# telling that access denied. So, we'll emulate 403 ourselves.
# TODO: currently, any non-csv result treated as 403.
@@ -143,7 +146,7 @@ class TracExport(object):
from html2text import html2text
url = self.full_url(self.TICKET_URL % id, 'rss')
self.log_url(url)
- d = feedparser.parse(url)
+ d = feedparser.parse(h.urlopen(url))
res = []
for comment in d['entries']:
c = {}
@@ -160,7 +163,7 @@ class TracExport(object):
# Scrape HTML to get ticket attachments
url = self.full_url(self.ATTACHMENT_LIST_URL % id)
self.log_url(url)
- f = urllib2.urlopen(url)
+ f = h.urlopen(url)
soup = BeautifulSoup(f)
attach = soup.find('div', id='attachments')
list = []
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_decorators.py b/Allura/allura/tests/test_decorators.py
new file mode 100644
index 0000000..893f327
--- /dev/null
+++ b/Allura/allura/tests/test_decorators.py
@@ -0,0 +1,39 @@
+from unittest import TestCase
+
+from mock import patch
+
+from allura.lib.decorators import task
+
+
+class TestTask(TestCase):
+
+ def test_no_params(self):
+ @task
+ def func():
+ pass
+ self.assertTrue(hasattr(func, 'post'))
+
+ def test_with_params(self):
+ @task(disable_notifications=True)
+ def func():
+ pass
+ self.assertTrue(hasattr(func, 'post'))
+
+ @patch('allura.lib.decorators.c')
+ @patch('allura.lib.decorators._get_model')
+ def test_post(self, c, _get_model):
+ @task(disable_notifications=True)
+ def func(s, foo=None, **kw):
+ pass
+ def mock_post(f, args, kw, delay=None):
+ self.assertTrue(c.project.notifications_disabled)
+ self.assertFalse('delay' in kw)
+ self.assertEqual(delay, 1)
+ self.assertEqual(kw, dict(foo=2))
+ self.assertEqual(args, ('test',))
+ self.assertEqual(f, func)
+
+ c.project.notifications_disabled = False
+ M = _get_model.return_value
+ M.MonQTask.post.side_effect = mock_post
+ func.post('test', foo=2, delay=1)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 42fb963..4ec91df 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -254,3 +254,45 @@ def test_datetimeformat():
def test_split_select_field_options():
assert_equals(h.split_select_field_options('"test message" test2'), ['test message', 'test2'])
assert_equals(h.split_select_field_options('"test message test2'), ['test', 'message', 'test2'])
+
+
+def test_notifications_disabled():
+ project = Mock(notifications_disabled=False)
+ with h.notifications_disabled(project):
+ assert_equals(project.notifications_disabled, True)
+ assert_equals(project.notifications_disabled, False)
+
+
+class TestUrlOpen(TestCase):
+ @patch('allura.lib.helpers.urllib2')
+ def test_no_error(self, urllib2):
+ r = h.urlopen('myurl')
+ self.assertEqual(r, urllib2.urlopen.return_value)
+ urllib2.urlopen.assert_called_once_with('myurl')
+
+ @patch('allura.lib.helpers.urllib2.urlopen')
+ def test_socket_timeout(self, urlopen):
+ import socket
+ def side_effect(url):
+ raise socket.timeout()
+ urlopen.side_effect = side_effect
+ self.assertRaises(socket.timeout, h.urlopen, 'myurl')
+ self.assertEqual(urlopen.call_count, 4)
+
+ @patch('allura.lib.helpers.urllib2.urlopen')
+ def test_handled_http_error(self, urlopen):
+ from urllib2 import HTTPError
+ def side_effect(url):
+ raise HTTPError('url', 408, 'timeout', None, None)
+ urlopen.side_effect = side_effect
+ self.assertRaises(HTTPError, h.urlopen, 'myurl')
+ self.assertEqual(urlopen.call_count, 4)
+
+ @patch('allura.lib.helpers.urllib2.urlopen')
+ def test_unhandled_http_error(self, urlopen):
+ from urllib2 import HTTPError
+ def side_effect(url):
+ raise HTTPError('url', 404, 'timeout', None, None)
+ urlopen.side_effect = side_effect
+ self.assertRaises(HTTPError, h.urlopen, 'myurl')
+ self.assertEqual(urlopen.call_count, 1)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_validators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_validators.py b/Allura/allura/tests/test_validators.py
index 8a40610..b0d165c 100644
--- a/Allura/allura/tests/test_validators.py
+++ b/Allura/allura/tests/test_validators.py
@@ -44,6 +44,42 @@ class TestJsonConverter(unittest.TestCase):
self.val.to_python('{')
+class TestJsonFile(unittest.TestCase):
+ val = v.JsonFile
+
+ class FieldStorage(object):
+ def __init__(self, content):
+ self.value = content
+
+ def test_valid(self):
+ self.assertEqual({}, self.val.to_python(self.FieldStorage('{}')))
+
+ def test_invalid(self):
+ with self.assertRaises(fe.Invalid):
+ self.val.to_python(self.FieldStorage('{'))
+
+
+class TestUserMapFile(unittest.TestCase):
+ val = v.UserMapJsonFile()
+
+ class FieldStorage(object):
+ def __init__(self, content):
+ self.value = content
+
+ def test_valid(self):
+ self.assertEqual({"user_old": "user_new"}, self.val.to_python(
+ self.FieldStorage('{"user_old": "user_new"}')))
+
+ def test_invalid(self):
+ with self.assertRaises(fe.Invalid):
+ self.val.to_python(self.FieldStorage('{"user_old": 1}'))
+
+ def test_as_string(self):
+ val = v.UserMapJsonFile(as_string=True)
+ self.assertEqual('{"user_old": "user_new"}', val.to_python(
+ self.FieldStorage('{"user_old": "user_new"}')))
+
+
class TestUserValidator(unittest.TestCase):
val = v.UserValidator
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeBlog/forgeblog/command/rssfeeds.py
----------------------------------------------------------------------
diff --git a/ForgeBlog/forgeblog/command/rssfeeds.py b/ForgeBlog/forgeblog/command/rssfeeds.py
index 9f44fd4..305bc5a 100644
--- a/ForgeBlog/forgeblog/command/rssfeeds.py
+++ b/ForgeBlog/forgeblog/command/rssfeeds.py
@@ -33,7 +33,7 @@ from forgeblog import model as BM
from forgeblog import version
from forgeblog.main import ForgeBlogApp
from allura.lib import exceptions
-from allura.lib.decorators import exceptionless
+from allura.lib.helpers import exceptionless
## Everything in this file depends on html2text,
## so import attempt is placed in global scope.
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 8cc3b52..e1af4f7 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -50,7 +50,7 @@ class ProjectImportForm(schema.Schema):
project_name = fev.UnicodeString(not_empty=True, max=40)
-@task
+@task(notifications_disabled=True)
def import_tool(importer_name, project_name=None, mount_point=None, mount_label=None, **kw):
importer = ToolImporter.by_name(importer_name)
importer.import_tool(c.project, c.user, project_name=project_name,
@@ -61,8 +61,9 @@ class ProjectImporter(BaseController):
"""
Base class for project importers.
- Subclases are required to implement the :meth:`index()` and
+ Subclasses are required to implement the :meth:`index()` and
:meth:`process()` views described below.
+
"""
source = None
process_validator = None
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/templates/project_base.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/templates/project_base.html b/ForgeImporters/forgeimporters/templates/project_base.html
index 6f1b683..9453e65 100644
--- a/ForgeImporters/forgeimporters/templates/project_base.html
+++ b/ForgeImporters/forgeimporters/templates/project_base.html
@@ -53,11 +53,10 @@
}
function check_names() {
- var data = {
- 'neighborhood': $('#neighborhood').val(),
- 'project_name': $('#project_name').val(),
- 'project_shortname': $('#project_shortname').val()
- };
+ var data = {};
+ $('#project-import-form input').each(function() {
+ data[$(this).attr('name')] = $(this).val();
+ });
$.getJSON('check_names', data, function(result) {
$('#project_name_error').addClass('hidden');
$('#project_shortname_error').addClass('hidden');
@@ -86,7 +85,7 @@
{% endblock %}
{% block content %}
-<form id="project-import-form" method="POST" action="process">
+<form id="project-import-form" method="POST" action="process" enctype="multipart/form-data">
<input type="hidden" id="neighborhood" name="neighborhood" value="{{importer.neighborhood.name}}"/>
<fieldset id="project-fields">
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/project.py b/ForgeImporters/forgeimporters/trac/project.py
index 66e8326..71ef5f4 100644
--- a/ForgeImporters/forgeimporters/trac/project.py
+++ b/ForgeImporters/forgeimporters/trac/project.py
@@ -23,6 +23,7 @@ from tg import expose, validate
from tg.decorators import with_trailing_slash
from allura.lib.decorators import require_post
+from allura.lib.validators import UserMapJsonFile
from .. import base
@@ -32,6 +33,7 @@ log = logging.getLogger(__name__)
class TracProjectForm(base.ProjectImportForm):
trac_url = fev.URL(not_empty=True)
+ user_map = UserMapJsonFile(as_string=True)
class TracProjectImporter(base.ProjectImporter):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/project.html b/ForgeImporters/forgeimporters/trac/templates/project.html
index 869e290..a9af734 100644
--- a/ForgeImporters/forgeimporters/trac/templates/project.html
+++ b/ForgeImporters/forgeimporters/trac/templates/project.html
@@ -24,9 +24,21 @@
</div>
<div class="grid-10">
<input id="trac_url" name="trac_url" value="{{c.form_values['trac_url']}}" autofocus/>
- <div id="trac_ur_errorl" class="error{% if not c.form_errors['trac_url'] %} hidden{% endif %}">
+ <div id="trac_url_error" class="error{% if not c.form_errors['trac_url'] %} hidden{% endif %}">
{{c.form_errors['trac_url']}}
</div>
</div>
+
+ <div class="grid-6" style="clear:left">
+ <label>User Map (optional)</label>
+ </div>
+ <div class="grid-10">
+ <input id="user_map" name="user_map" value="{{c.form_values['user_map']}}" type="file"/>
+ <br><small>JSON file mapping Trac usernames to Allura usernames</small>
+ <div id="user_map_error" class="error{% if not c.form_errors['user_map'] %} hidden{% endif %}">
+ {{c.form_errors['user_map']}}
+ </div>
+ </div>
+
{{ super() }}
{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/tickets/index.html b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
index eaf9aac..7c278b0 100644
--- a/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
+++ b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
@@ -27,10 +27,13 @@ Import tickets from Trac
{% endblock %}
{% block content %}
-<form action="create" method="post" class="pad">
+<form action="create" method="post" enctype="multipart/form-data" class="pad">
<label for="trac_url">URL of the Trac instance</label>
<input name="trac_url" />
+ <label for="user_map">JSON file mapping Trac usernames to Allura usernames (optional)</label>
+ <input name="user_map" type="file" />
+
<label for="mount_label">Label</label>
<input name="mount_label" value="Source" />
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 5f88eef..2a539df 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import json
from unittest import TestCase
from mock import Mock, patch
@@ -39,7 +40,7 @@ class TestTracTicketImporter(TestCase):
from datetime import datetime, timedelta
now = datetime.utcnow()
dt.utcnow.return_value = now
-
+ user_map = {"orig_user":"new_user"}
importer = TracTicketImporter()
app = Mock(name='ForgeTrackerApp')
project = Mock(name='Project', shortname='myproject')
@@ -48,7 +49,9 @@ class TestTracTicketImporter(TestCase):
res = importer.import_tool(project, user,
mount_point='bugs',
mount_label='Bugs',
- trac_url='http://example.com/trac/url')
+ trac_url='http://example.com/trac/url',
+ user_map=json.dumps(user_map),
+ )
self.assertEqual(res, app)
project.install_app.assert_called_once_with(
'Tickets', mount_point='bugs', mount_label='Bugs')
@@ -60,7 +63,8 @@ class TestTracTicketImporter(TestCase):
expires=now + timedelta(minutes=60))
api_client = ApiClient.return_value
import_tracker.assert_called_once_with(
- api_client, 'myproject', 'bugs', {}, '[]',
+ api_client, 'myproject', 'bugs',
+ {"user_map": user_map}, '[]',
validate=False)
g.post_event.assert_called_once_with('project_updated')
@@ -91,10 +95,12 @@ class TestTracTicketImportController(TestController, TestCase):
mount_point='mymount',
)
r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+ upload_files=[('user_map', 'myfile', '{"orig_user": "new_user"}')],
status=302)
project = M.Project.query.get(shortname='test')
self.assertEqual(r.location, 'http://localhost/p/test/mymount')
self.assertEqual(project._id, importer.import_tool.call_args[0][0]._id)
self.assertEqual(u'mymount', importer.import_tool.call_args[1]['mount_point'])
self.assertEqual(u'mylabel', importer.import_tool.call_args[1]['mount_label'])
+ self.assertEqual('{"orig_user": "new_user"}', importer.import_tool.call_args[1]['user_map'])
self.assertEqual(u'http://example.com/trac/url', importer.import_tool.call_args[1]['trac_url'])
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index f7d50b4..00bca2a 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -41,6 +41,7 @@ from tg.decorators import (
from allura.controllers import BaseController
from allura.lib.decorators import require_post
from allura.lib.import_api import AlluraImportApiClient
+from allura.lib.validators import UserMapJsonFile
from allura.model import ApiTicket
from allura.scripts.trac_export import (
TracExport,
@@ -54,6 +55,7 @@ from forgetracker.scripts.import_tracker import import_tracker
class TracTicketImportSchema(fe.Schema):
trac_url = fev.URL(not_empty=True)
+ user_map = UserMapJsonFile(as_string=True)
mount_point = fev.UnicodeString()
mount_label = fev.UnicodeString()
@@ -68,11 +70,12 @@ class TracTicketImportController(BaseController):
@expose()
@require_post()
@validate(TracTicketImportSchema(), error_handler=index)
- def create(self, trac_url, mount_point, mount_label, **kw):
+ def create(self, trac_url, mount_point, mount_label, user_map=None, **kw):
app = TracTicketImporter().import_tool(c.project, c.user,
mount_point=mount_point,
mount_label=mount_label,
- trac_url=trac_url)
+ trac_url=trac_url,
+ user_map=user_map)
redirect(app.url())
@@ -84,7 +87,7 @@ class TracTicketImporter(ToolImporter):
tool_description = 'Import your tickets from Trac'
def import_tool(self, project, user, project_name=None, mount_point=None,
- mount_label=None, trac_url=None, **kw):
+ mount_label=None, trac_url=None, user_map=None, **kw):
""" Import Trac tickets into a new Allura Tracker tool.
"""
@@ -105,7 +108,8 @@ class TracTicketImporter(ToolImporter):
session(api_ticket).flush(api_ticket)
cli = AlluraImportApiClient(config['base_url'], api_ticket.api_key,
api_ticket.secret_key, False)
- import_tracker(cli, project.shortname, mount_point, {},
+ import_tracker(cli, project.shortname, mount_point,
+ {'user_map': json.loads(user_map) if user_map else {}},
export_string, validate=False)
g.post_event('project_updated')
return app
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeTracker/forgetracker/import_support.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/import_support.py b/ForgeTracker/forgetracker/import_support.py
index 26e182e..04d453a 100644
--- a/ForgeTracker/forgetracker/import_support.py
+++ b/ForgeTracker/forgetracker/import_support.py
@@ -136,11 +136,11 @@ class ImportSupport(object):
return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
def get_user_id(self, username):
- username = self.options['user_map'].get(username, username)
+ username = self.options['user_map'].get(username)
+ if not username:
+ return None
u = M.User.by_username(username)
- if u:
- return u._id
- return None
+ return u._id if u else None
def check_custom_field(self, field, value):
field = c.app.globals.get_custom_field(field)
@@ -193,7 +193,12 @@ class ImportSupport(object):
new_f, conv = transform
remapped[new_f] = conv(v)
- remapped['description'] = self.link_processing(remapped['description'])
+ description = self.link_processing(remapped['description'])
+ if ticket_dict['submitter'] and not remapped['reported_by_id']:
+ description = 'Originally created by: {0}\n\n{1}'.format(
+ ticket_dict['submitter'], description)
+ remapped['description'] = description
+
ticket_num = ticket_dict['id']
existing_ticket = TM.Ticket.query.get(app_config_id=c.app.config._id,
ticket_num=ticket_num)
@@ -261,8 +266,13 @@ class ImportSupport(object):
def make_comment(self, thread, comment_dict):
ts = self.parse_date(comment_dict['date'])
- comment = thread.post(text=self.link_processing(comment_dict['comment']), timestamp=ts)
- comment.author_id = self.get_user_id(comment_dict['submitter'])
+ author_id = self.get_user_id(comment_dict['submitter'])
+ text = self.link_processing(comment_dict['comment'])
+ if not author_id and comment_dict['submitter']:
+ text = 'Originally posted by: {0}\n\n{1}'.format(
+ comment_dict['submitter'], text)
+ comment = thread.post(text=text, timestamp=ts)
+ comment.author_id = author_id
comment.import_id = c.api_token.api_key
def make_attachment(self, org_ticket_id, ticket_id, att_dict):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeTracker/forgetracker/tracker_main.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index a1363dc..2b432df 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -1608,18 +1608,19 @@ class RootRestController(BaseController):
@expose('json:')
def perform_import(self, doc=None, options=None, **post_data):
- require_access(c.project, 'admin')
- if c.api_token.get_capability('import') != [c.project.neighborhood.name, c.project.shortname]:
- log.error('Import capability is not enabled for %s', c.project.shortname)
- raise exc.HTTPForbidden(detail='Import is not allowed')
+ with h.notifications_disabled(c.project):
+ require_access(c.project, 'admin')
+ if c.api_token.get_capability('import') != [c.project.neighborhood.name, c.project.shortname]:
+ log.error('Import capability is not enabled for %s', c.project.shortname)
+ raise exc.HTTPForbidden(detail='Import is not allowed')
- migrator = ImportSupport()
- try:
- status = migrator.perform_import(doc, options, **post_data)
- return status
- except Exception, e:
- log.exception(e)
- return dict(status=False, errors=[str(e)])
+ migrator = ImportSupport()
+ try:
+ status = migrator.perform_import(doc, options, **post_data)
+ return status
+ except Exception, e:
+ log.exception(e)
+ return dict(status=False, errors=[str(e)])
@expose('json:')
def search(self, q=None, limit=100, page=0, sort=None, **kw):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeWiki/forgewiki/wiki_main.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/wiki_main.py b/ForgeWiki/forgewiki/wiki_main.py
index 816db78..e059038 100644
--- a/ForgeWiki/forgewiki/wiki_main.py
+++ b/ForgeWiki/forgewiki/wiki_main.py
@@ -730,17 +730,18 @@ class PageRestController(BaseController):
return self.page.__json__()
def _update_page(self, title, **post_data):
- if not self.page:
- require_access(c.app, 'create')
- self.page = WM.Page.upsert(title)
- self.page.viewable_by = ['all']
- else:
- require_access(self.page, 'edit')
- self.page.text = post_data['text']
- if 'labels' in post_data:
- self.page.labels = post_data['labels'].split(',')
- self.page.commit()
- return {}
+ with h.notifications_disabled(c.project):
+ if not self.page:
+ require_access(c.app, 'create')
+ self.page = WM.Page.upsert(title)
+ self.page.viewable_by = ['all']
+ else:
+ require_access(self.page, 'edit')
+ self.page.text = post_data['text']
+ if 'labels' in post_data:
+ self.page.labels = post_data['labels'].split(',')
+ self.page.commit()
+ return {}
class WikiAdminController(DefaultAdminController):