You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/08/16 02:30:04 UTC
[1/6] git commit: [#6480] Improvements to Trac importer
Updated Branches:
refs/heads/master 6657808db -> 3e67aa667
[#6480] Improvements to Trac importer
- Disable email notifications during import
- Retry time-out HTTP requests
- Add option to supply user-map file
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/675495e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/675495e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/675495e3
Branch: refs/heads/master
Commit: 675495e39723e593cba00cdcf84e3cb02d00e4b0
Parents: 6657808
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Mon Aug 12 18:06:43 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Thu Aug 15 16:08:41 2013 +0000
----------------------------------------------------------------------
Allura/allura/lib/decorators.py | 75 +++++++++++---------
Allura/allura/lib/helpers.py | 71 +++++++++++++++++-
Allura/allura/lib/import_api.py | 2 +-
Allura/allura/lib/spam/__init__.py | 2 +-
Allura/allura/lib/validators.py | 30 ++++++++
Allura/allura/scripts/trac_export.py | 9 ++-
Allura/allura/tests/test_decorators.py | 39 ++++++++++
Allura/allura/tests/test_helpers.py | 42 +++++++++++
Allura/allura/tests/test_validators.py | 36 ++++++++++
ForgeBlog/forgeblog/command/rssfeeds.py | 2 +-
ForgeImporters/forgeimporters/base.py | 5 +-
.../forgeimporters/templates/project_base.html | 11 ++-
ForgeImporters/forgeimporters/trac/project.py | 2 +
.../forgeimporters/trac/templates/project.html | 14 +++-
.../trac/templates/tickets/index.html | 5 +-
.../forgeimporters/trac/tests/test_tickets.py | 12 +++-
ForgeImporters/forgeimporters/trac/tickets.py | 12 ++--
ForgeTracker/forgetracker/import_support.py | 24 +++++--
ForgeTracker/forgetracker/tracker_main.py | 23 +++---
ForgeWiki/forgewiki/wiki_main.py | 23 +++---
20 files changed, 354 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index 36fb2de..137152e 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -28,16 +28,49 @@ from tg import request, redirect
from webob import exc
-def task(func):
- '''Decorator to add some methods to task functions'''
- def post(*args, **kwargs):
- from allura import model as M
- delay = kwargs.pop('delay', 0)
- return M.MonQTask.post(func, args, kwargs, delay=delay)
- # if decorating a class, have to make it a staticmethod
- # or it gets a spurious cls argument
- func.post = staticmethod(post) if inspect.isclass(func) else post
- return func
+from pylons import tmpl_context as c
+from allura.lib import helpers as h
+
+
+def _get_model():
+ from allura import model as M
+ return M
+
+def task(*args, **kw):
+ """Decorator that adds a ``.post()`` function to the decorated callable.
+
+ Calling <original_callable>.post(*args, **kw) queues the callable for
+ execution by a background worker process. All parameters must be
+ BSON-serializable.
+
+ Example usage:
+
+ @task
+ def myfunc():
+ pass
+
+ @task(notifications_disabled=True)
+ def myotherfunc():
+ # No email notifications will be sent for c.project during this task
+ pass
+
+ """
+ def task_(func):
+ def post(*args, **kwargs):
+ delay = kwargs.pop('delay', 0)
+ project = getattr(c, 'project', None)
+ cm = (h.notifications_disabled if project and
+ kw.get('notifications_disabled') else h.null_contextmanager)
+ with cm(project):
+ M = _get_model()
+ return M.MonQTask.post(func, args, kwargs, delay=delay)
+ # if decorating a class, have to make it a staticmethod
+ # or it gets a spurious cls argument
+ func.post = staticmethod(post) if inspect.isclass(func) else post
+ return func
+ if len(args) == 1 and callable(args[0]):
+ return task_(args[0])
+ return task_
class event_handler(object):
'''Decorator to register event handlers'''
@@ -155,28 +188,6 @@ class log_action(object): # pragma no cover
extra['referer_link'] = referer_link
return extra
-class exceptionless(object):
- '''Decorator making the decorated function return 'error_result' on any
- exceptions rather than propagating exceptions up the stack
- '''
-
- def __init__(self, error_result, log=None):
- self.error_result = error_result
- self.log = log
-
- def __call__(self, fun):
- fname = 'exceptionless(%s)' % fun.__name__
- def inner(*args, **kwargs):
- try:
- return fun(*args, **kwargs)
- except Exception as e:
- if self.log:
- self.log.exception('Error calling %s(args=%s, kwargs=%s): %s',
- fname, args, kwargs, str(e))
- return self.error_result
- inner.__name__ = fname
- return inner
-
def Property(function):
'''Decorator to easily assign descriptors based on sub-function names
See <http://code.activestate.com/recipes/410698-property-decorator-for-python-24/>
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index 1604aa9..0511dc7 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -22,6 +22,7 @@ import os
import os.path
import difflib
import urllib
+import urllib2
import re
import json
import logging
@@ -30,6 +31,7 @@ from hashlib import sha1
from datetime import datetime, timedelta
from collections import defaultdict
import shlex
+import socket
import tg
import genshi.template
@@ -53,10 +55,10 @@ from webhelpers import date, feedgenerator, html, number, misc, text
from allura.lib import exceptions as exc
# Reimport to make available to templates
-from allura.lib.decorators import exceptionless
from allura.lib import AsciiDammit
from .security import has_access
+log = logging.getLogger(__name__)
# validates project, subproject, and user names
re_project_name = re.compile(r'^[a-z][-a-z0-9]{2,14}$')
@@ -857,3 +859,70 @@ def split_select_field_options(field_options):
# so we're getting rid of those.
field_options = [o.replace('"', '') for o in field_options]
return field_options
+
+
+@contextmanager
+def notifications_disabled(project):
+ """Temporarily disable email notifications on a project.
+
+ """
+ orig = project.notifications_disabled
+ try:
+ project.notifications_disabled = True
+ yield
+ finally:
+ project.notifications_disabled = orig
+
+
+@contextmanager
+def null_contextmanager(*args, **kw):
+ """A no-op contextmanager.
+
+ """
+ yield
+
+
+class exceptionless(object):
+ '''Decorator making the decorated function return 'error_result' on any
+ exceptions rather than propagating exceptions up the stack
+ '''
+
+ def __init__(self, error_result, log=None):
+ self.error_result = error_result
+ self.log = log
+
+ def __call__(self, fun):
+ fname = 'exceptionless(%s)' % fun.__name__
+ def inner(*args, **kwargs):
+ try:
+ return fun(*args, **kwargs)
+ except Exception as e:
+ if self.log:
+ self.log.exception('Error calling %s(args=%s, kwargs=%s): %s',
+ fname, args, kwargs, str(e))
+ return self.error_result
+ inner.__name__ = fname
+ return inner
+
+
+def urlopen(url, retries=3, codes=(408,)):
+ """Open url, optionally retrying if an error is encountered.
+
+ Socket timeouts will always be retried if retries > 0.
+ HTTP errors are retried if the error code is passed in ``codes``.
+
+ :param retries: Number of time to retry.
+ :param codes: HTTP error codes that should be retried.
+
+ """
+ while True:
+ try:
+ return urllib2.urlopen(url)
+ except (urllib2.HTTPError, socket.timeout) as e:
+ if retries and (isinstance(e, socket.timeout) or
+ e.code in codes):
+ retries -= 1
+ continue
+ else:
+ log.exception('Failed after %s retries: %s', retries, e)
+ raise
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/import_api.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/import_api.py b/Allura/allura/lib/import_api.py
index 1f3f98d..8eeb8a7 100644
--- a/Allura/allura/lib/import_api.py
+++ b/Allura/allura/lib/import_api.py
@@ -49,7 +49,7 @@ class AlluraImportApiClient(object):
url = urlparse.urljoin(self.base_url, url)
if self.verbose:
print "Using URL '%s'" % (url)
-
+
params = self.sign(urlparse.urlparse(url).path, params.items())
while True:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/spam/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/__init__.py b/Allura/allura/lib/spam/__init__.py
index 8e6607a..cfb5c41 100644
--- a/Allura/allura/lib/spam/__init__.py
+++ b/Allura/allura/lib/spam/__init__.py
@@ -17,7 +17,7 @@
import logging
-from allura.lib.decorators import exceptionless
+from allura.lib.helpers import exceptionless
log = logging.getLogger(__name__)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/lib/validators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/validators.py b/Allura/allura/lib/validators.py
index ae863e8..9b44d53 100644
--- a/Allura/allura/lib/validators.py
+++ b/Allura/allura/lib/validators.py
@@ -151,6 +151,36 @@ class JsonConverter(fev.FancyValidator):
raise fe.Invalid('Invalid JSON: ' + str(e), value, state)
return obj
+class JsonFile(fev.FieldStorageUploadConverter):
+ """Validates that a file is JSON and returns the deserialized Python object
+
+ """
+ def _to_python(self, value, state):
+ return JsonConverter.to_python(value.value)
+
+class UserMapJsonFile(JsonFile):
+ """Validates that a JSON file conforms to this format:
+
+ {str:str, ...}
+
+ and returns a deserialized or stringified copy of it.
+
+ """
+ def __init__(self, as_string=False):
+ self.as_string = as_string
+
+ def _to_python(self, value, state):
+ value = super(self.__class__, self)._to_python(value, state)
+ try:
+ for k, v in value.iteritems():
+ if not(isinstance(k, basestring) and isinstance(v, basestring)):
+ raise
+ return json.dumps(value) if self.as_string else value
+ except:
+ raise fe.Invalid(
+ 'User map file must contain mapping of {str:str, ...}',
+ value, state)
+
class CreateTaskSchema(fe.Schema):
task = TaskValidator(not_empty=True, strip=True)
task_args = JsonConverter(if_missing=dict(args=[], kwargs={}))
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
index e45175b..d53afbc 100644
--- a/Allura/allura/scripts/trac_export.py
+++ b/Allura/allura/scripts/trac_export.py
@@ -18,6 +18,7 @@
# under the License.
import logging
+import socket
import sys
import csv
import urlparse
@@ -34,6 +35,8 @@ from BeautifulSoup import BeautifulSoup, NavigableString
import dateutil.parser
import pytz
+from allura.lib import helpers as h
+
log = logging.getLogger(__name__)
@@ -121,7 +124,7 @@ class TracExport(object):
def csvopen(self, url):
self.log_url(url)
- f = urllib2.urlopen(url)
+ f = h.urlopen(url)
# Trac doesn't throw 403 error, just shows normal 200 HTML page
# telling that access denied. So, we'll emulate 403 ourselves.
# TODO: currently, any non-csv result treated as 403.
@@ -143,7 +146,7 @@ class TracExport(object):
from html2text import html2text
url = self.full_url(self.TICKET_URL % id, 'rss')
self.log_url(url)
- d = feedparser.parse(url)
+ d = feedparser.parse(h.urlopen(url))
res = []
for comment in d['entries']:
c = {}
@@ -160,7 +163,7 @@ class TracExport(object):
# Scrape HTML to get ticket attachments
url = self.full_url(self.ATTACHMENT_LIST_URL % id)
self.log_url(url)
- f = urllib2.urlopen(url)
+ f = h.urlopen(url)
soup = BeautifulSoup(f)
attach = soup.find('div', id='attachments')
list = []
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_decorators.py b/Allura/allura/tests/test_decorators.py
new file mode 100644
index 0000000..893f327
--- /dev/null
+++ b/Allura/allura/tests/test_decorators.py
@@ -0,0 +1,39 @@
+from unittest import TestCase
+
+from mock import patch
+
+from allura.lib.decorators import task
+
+
+class TestTask(TestCase):
+
+ def test_no_params(self):
+ @task
+ def func():
+ pass
+ self.assertTrue(hasattr(func, 'post'))
+
+ def test_with_params(self):
+ @task(disable_notifications=True)
+ def func():
+ pass
+ self.assertTrue(hasattr(func, 'post'))
+
+ @patch('allura.lib.decorators.c')
+ @patch('allura.lib.decorators._get_model')
+ def test_post(self, c, _get_model):
+ @task(disable_notifications=True)
+ def func(s, foo=None, **kw):
+ pass
+ def mock_post(f, args, kw, delay=None):
+ self.assertTrue(c.project.notifications_disabled)
+ self.assertFalse('delay' in kw)
+ self.assertEqual(delay, 1)
+ self.assertEqual(kw, dict(foo=2))
+ self.assertEqual(args, ('test',))
+ self.assertEqual(f, func)
+
+ c.project.notifications_disabled = False
+ M = _get_model.return_value
+ M.MonQTask.post.side_effect = mock_post
+ func.post('test', foo=2, delay=1)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 42fb963..4ec91df 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -254,3 +254,45 @@ def test_datetimeformat():
def test_split_select_field_options():
assert_equals(h.split_select_field_options('"test message" test2'), ['test message', 'test2'])
assert_equals(h.split_select_field_options('"test message test2'), ['test', 'message', 'test2'])
+
+
+def test_notifications_disabled():
+ project = Mock(notifications_disabled=False)
+ with h.notifications_disabled(project):
+ assert_equals(project.notifications_disabled, True)
+ assert_equals(project.notifications_disabled, False)
+
+
+class TestUrlOpen(TestCase):
+ @patch('allura.lib.helpers.urllib2')
+ def test_no_error(self, urllib2):
+ r = h.urlopen('myurl')
+ self.assertEqual(r, urllib2.urlopen.return_value)
+ urllib2.urlopen.assert_called_once_with('myurl')
+
+ @patch('allura.lib.helpers.urllib2.urlopen')
+ def test_socket_timeout(self, urlopen):
+ import socket
+ def side_effect(url):
+ raise socket.timeout()
+ urlopen.side_effect = side_effect
+ self.assertRaises(socket.timeout, h.urlopen, 'myurl')
+ self.assertEqual(urlopen.call_count, 4)
+
+ @patch('allura.lib.helpers.urllib2.urlopen')
+ def test_handled_http_error(self, urlopen):
+ from urllib2 import HTTPError
+ def side_effect(url):
+ raise HTTPError('url', 408, 'timeout', None, None)
+ urlopen.side_effect = side_effect
+ self.assertRaises(HTTPError, h.urlopen, 'myurl')
+ self.assertEqual(urlopen.call_count, 4)
+
+ @patch('allura.lib.helpers.urllib2.urlopen')
+ def test_unhandled_http_error(self, urlopen):
+ from urllib2 import HTTPError
+ def side_effect(url):
+ raise HTTPError('url', 404, 'timeout', None, None)
+ urlopen.side_effect = side_effect
+ self.assertRaises(HTTPError, h.urlopen, 'myurl')
+ self.assertEqual(urlopen.call_count, 1)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/Allura/allura/tests/test_validators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_validators.py b/Allura/allura/tests/test_validators.py
index 8a40610..b0d165c 100644
--- a/Allura/allura/tests/test_validators.py
+++ b/Allura/allura/tests/test_validators.py
@@ -44,6 +44,42 @@ class TestJsonConverter(unittest.TestCase):
self.val.to_python('{')
+class TestJsonFile(unittest.TestCase):
+ val = v.JsonFile
+
+ class FieldStorage(object):
+ def __init__(self, content):
+ self.value = content
+
+ def test_valid(self):
+ self.assertEqual({}, self.val.to_python(self.FieldStorage('{}')))
+
+ def test_invalid(self):
+ with self.assertRaises(fe.Invalid):
+ self.val.to_python(self.FieldStorage('{'))
+
+
+class TestUserMapFile(unittest.TestCase):
+ val = v.UserMapJsonFile()
+
+ class FieldStorage(object):
+ def __init__(self, content):
+ self.value = content
+
+ def test_valid(self):
+ self.assertEqual({"user_old": "user_new"}, self.val.to_python(
+ self.FieldStorage('{"user_old": "user_new"}')))
+
+ def test_invalid(self):
+ with self.assertRaises(fe.Invalid):
+ self.val.to_python(self.FieldStorage('{"user_old": 1}'))
+
+ def test_as_string(self):
+ val = v.UserMapJsonFile(as_string=True)
+ self.assertEqual('{"user_old": "user_new"}', val.to_python(
+ self.FieldStorage('{"user_old": "user_new"}')))
+
+
class TestUserValidator(unittest.TestCase):
val = v.UserValidator
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeBlog/forgeblog/command/rssfeeds.py
----------------------------------------------------------------------
diff --git a/ForgeBlog/forgeblog/command/rssfeeds.py b/ForgeBlog/forgeblog/command/rssfeeds.py
index 9f44fd4..305bc5a 100644
--- a/ForgeBlog/forgeblog/command/rssfeeds.py
+++ b/ForgeBlog/forgeblog/command/rssfeeds.py
@@ -33,7 +33,7 @@ from forgeblog import model as BM
from forgeblog import version
from forgeblog.main import ForgeBlogApp
from allura.lib import exceptions
-from allura.lib.decorators import exceptionless
+from allura.lib.helpers import exceptionless
## Everything in this file depends on html2text,
## so import attempt is placed in global scope.
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 8cc3b52..e1af4f7 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -50,7 +50,7 @@ class ProjectImportForm(schema.Schema):
project_name = fev.UnicodeString(not_empty=True, max=40)
-@task
+@task(notifications_disabled=True)
def import_tool(importer_name, project_name=None, mount_point=None, mount_label=None, **kw):
importer = ToolImporter.by_name(importer_name)
importer.import_tool(c.project, c.user, project_name=project_name,
@@ -61,8 +61,9 @@ class ProjectImporter(BaseController):
"""
Base class for project importers.
- Subclases are required to implement the :meth:`index()` and
+ Subclasses are required to implement the :meth:`index()` and
:meth:`process()` views described below.
+
"""
source = None
process_validator = None
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/templates/project_base.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/templates/project_base.html b/ForgeImporters/forgeimporters/templates/project_base.html
index 6f1b683..9453e65 100644
--- a/ForgeImporters/forgeimporters/templates/project_base.html
+++ b/ForgeImporters/forgeimporters/templates/project_base.html
@@ -53,11 +53,10 @@
}
function check_names() {
- var data = {
- 'neighborhood': $('#neighborhood').val(),
- 'project_name': $('#project_name').val(),
- 'project_shortname': $('#project_shortname').val()
- };
+ var data = {};
+ $('#project-import-form input').each(function() {
+ data[$(this).attr('name')] = $(this).val();
+ });
$.getJSON('check_names', data, function(result) {
$('#project_name_error').addClass('hidden');
$('#project_shortname_error').addClass('hidden');
@@ -86,7 +85,7 @@
{% endblock %}
{% block content %}
-<form id="project-import-form" method="POST" action="process">
+<form id="project-import-form" method="POST" action="process" enctype="multipart/form-data">
<input type="hidden" id="neighborhood" name="neighborhood" value="{{importer.neighborhood.name}}"/>
<fieldset id="project-fields">
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/project.py b/ForgeImporters/forgeimporters/trac/project.py
index 66e8326..71ef5f4 100644
--- a/ForgeImporters/forgeimporters/trac/project.py
+++ b/ForgeImporters/forgeimporters/trac/project.py
@@ -23,6 +23,7 @@ from tg import expose, validate
from tg.decorators import with_trailing_slash
from allura.lib.decorators import require_post
+from allura.lib.validators import UserMapJsonFile
from .. import base
@@ -32,6 +33,7 @@ log = logging.getLogger(__name__)
class TracProjectForm(base.ProjectImportForm):
trac_url = fev.URL(not_empty=True)
+ user_map = UserMapJsonFile(as_string=True)
class TracProjectImporter(base.ProjectImporter):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/project.html b/ForgeImporters/forgeimporters/trac/templates/project.html
index 869e290..a9af734 100644
--- a/ForgeImporters/forgeimporters/trac/templates/project.html
+++ b/ForgeImporters/forgeimporters/trac/templates/project.html
@@ -24,9 +24,21 @@
</div>
<div class="grid-10">
<input id="trac_url" name="trac_url" value="{{c.form_values['trac_url']}}" autofocus/>
- <div id="trac_ur_errorl" class="error{% if not c.form_errors['trac_url'] %} hidden{% endif %}">
+ <div id="trac_url_error" class="error{% if not c.form_errors['trac_url'] %} hidden{% endif %}">
{{c.form_errors['trac_url']}}
</div>
</div>
+
+ <div class="grid-6" style="clear:left">
+ <label>User Map (optional)</label>
+ </div>
+ <div class="grid-10">
+ <input id="user_map" name="user_map" value="{{c.form_values['user_map']}}" type="file"/>
+ <br><small>JSON file mapping Trac usernames to Allura usernames</small>
+ <div id="user_map_error" class="error{% if not c.form_errors['user_map'] %} hidden{% endif %}">
+ {{c.form_errors['user_map']}}
+ </div>
+ </div>
+
{{ super() }}
{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/tickets/index.html b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
index eaf9aac..7c278b0 100644
--- a/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
+++ b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
@@ -27,10 +27,13 @@ Import tickets from Trac
{% endblock %}
{% block content %}
-<form action="create" method="post" class="pad">
+<form action="create" method="post" enctype="multipart/form-data" class="pad">
<label for="trac_url">URL of the Trac instance</label>
<input name="trac_url" />
+ <label for="user_map">JSON file mapping Trac usernames to Allura usernames (optional)</label>
+ <input name="user_map" type="file" />
+
<label for="mount_label">Label</label>
<input name="mount_label" value="Source" />
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 5f88eef..2a539df 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import json
from unittest import TestCase
from mock import Mock, patch
@@ -39,7 +40,7 @@ class TestTracTicketImporter(TestCase):
from datetime import datetime, timedelta
now = datetime.utcnow()
dt.utcnow.return_value = now
-
+ user_map = {"orig_user":"new_user"}
importer = TracTicketImporter()
app = Mock(name='ForgeTrackerApp')
project = Mock(name='Project', shortname='myproject')
@@ -48,7 +49,9 @@ class TestTracTicketImporter(TestCase):
res = importer.import_tool(project, user,
mount_point='bugs',
mount_label='Bugs',
- trac_url='http://example.com/trac/url')
+ trac_url='http://example.com/trac/url',
+ user_map=json.dumps(user_map),
+ )
self.assertEqual(res, app)
project.install_app.assert_called_once_with(
'Tickets', mount_point='bugs', mount_label='Bugs')
@@ -60,7 +63,8 @@ class TestTracTicketImporter(TestCase):
expires=now + timedelta(minutes=60))
api_client = ApiClient.return_value
import_tracker.assert_called_once_with(
- api_client, 'myproject', 'bugs', {}, '[]',
+ api_client, 'myproject', 'bugs',
+ {"user_map": user_map}, '[]',
validate=False)
g.post_event.assert_called_once_with('project_updated')
@@ -91,10 +95,12 @@ class TestTracTicketImportController(TestController, TestCase):
mount_point='mymount',
)
r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+ upload_files=[('user_map', 'myfile', '{"orig_user": "new_user"}')],
status=302)
project = M.Project.query.get(shortname='test')
self.assertEqual(r.location, 'http://localhost/p/test/mymount')
self.assertEqual(project._id, importer.import_tool.call_args[0][0]._id)
self.assertEqual(u'mymount', importer.import_tool.call_args[1]['mount_point'])
self.assertEqual(u'mylabel', importer.import_tool.call_args[1]['mount_label'])
+ self.assertEqual('{"orig_user": "new_user"}', importer.import_tool.call_args[1]['user_map'])
self.assertEqual(u'http://example.com/trac/url', importer.import_tool.call_args[1]['trac_url'])
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index f7d50b4..00bca2a 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -41,6 +41,7 @@ from tg.decorators import (
from allura.controllers import BaseController
from allura.lib.decorators import require_post
from allura.lib.import_api import AlluraImportApiClient
+from allura.lib.validators import UserMapJsonFile
from allura.model import ApiTicket
from allura.scripts.trac_export import (
TracExport,
@@ -54,6 +55,7 @@ from forgetracker.scripts.import_tracker import import_tracker
class TracTicketImportSchema(fe.Schema):
trac_url = fev.URL(not_empty=True)
+ user_map = UserMapJsonFile(as_string=True)
mount_point = fev.UnicodeString()
mount_label = fev.UnicodeString()
@@ -68,11 +70,12 @@ class TracTicketImportController(BaseController):
@expose()
@require_post()
@validate(TracTicketImportSchema(), error_handler=index)
- def create(self, trac_url, mount_point, mount_label, **kw):
+ def create(self, trac_url, mount_point, mount_label, user_map=None, **kw):
app = TracTicketImporter().import_tool(c.project, c.user,
mount_point=mount_point,
mount_label=mount_label,
- trac_url=trac_url)
+ trac_url=trac_url,
+ user_map=user_map)
redirect(app.url())
@@ -84,7 +87,7 @@ class TracTicketImporter(ToolImporter):
tool_description = 'Import your tickets from Trac'
def import_tool(self, project, user, project_name=None, mount_point=None,
- mount_label=None, trac_url=None, **kw):
+ mount_label=None, trac_url=None, user_map=None, **kw):
""" Import Trac tickets into a new Allura Tracker tool.
"""
@@ -105,7 +108,8 @@ class TracTicketImporter(ToolImporter):
session(api_ticket).flush(api_ticket)
cli = AlluraImportApiClient(config['base_url'], api_ticket.api_key,
api_ticket.secret_key, False)
- import_tracker(cli, project.shortname, mount_point, {},
+ import_tracker(cli, project.shortname, mount_point,
+ {'user_map': json.loads(user_map) if user_map else {}},
export_string, validate=False)
g.post_event('project_updated')
return app
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeTracker/forgetracker/import_support.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/import_support.py b/ForgeTracker/forgetracker/import_support.py
index 26e182e..04d453a 100644
--- a/ForgeTracker/forgetracker/import_support.py
+++ b/ForgeTracker/forgetracker/import_support.py
@@ -136,11 +136,11 @@ class ImportSupport(object):
return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
def get_user_id(self, username):
- username = self.options['user_map'].get(username, username)
+ username = self.options['user_map'].get(username)
+ if not username:
+ return None
u = M.User.by_username(username)
- if u:
- return u._id
- return None
+ return u._id if u else None
def check_custom_field(self, field, value):
field = c.app.globals.get_custom_field(field)
@@ -193,7 +193,12 @@ class ImportSupport(object):
new_f, conv = transform
remapped[new_f] = conv(v)
- remapped['description'] = self.link_processing(remapped['description'])
+ description = self.link_processing(remapped['description'])
+ if ticket_dict['submitter'] and not remapped['reported_by_id']:
+ description = 'Originally created by: {0}\n\n{1}'.format(
+ ticket_dict['submitter'], description)
+ remapped['description'] = description
+
ticket_num = ticket_dict['id']
existing_ticket = TM.Ticket.query.get(app_config_id=c.app.config._id,
ticket_num=ticket_num)
@@ -261,8 +266,13 @@ class ImportSupport(object):
def make_comment(self, thread, comment_dict):
ts = self.parse_date(comment_dict['date'])
- comment = thread.post(text=self.link_processing(comment_dict['comment']), timestamp=ts)
- comment.author_id = self.get_user_id(comment_dict['submitter'])
+ author_id = self.get_user_id(comment_dict['submitter'])
+ text = self.link_processing(comment_dict['comment'])
+ if not author_id and comment_dict['submitter']:
+ text = 'Originally posted by: {0}\n\n{1}'.format(
+ comment_dict['submitter'], text)
+ comment = thread.post(text=text, timestamp=ts)
+ comment.author_id = author_id
comment.import_id = c.api_token.api_key
def make_attachment(self, org_ticket_id, ticket_id, att_dict):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeTracker/forgetracker/tracker_main.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/tracker_main.py b/ForgeTracker/forgetracker/tracker_main.py
index a1363dc..2b432df 100644
--- a/ForgeTracker/forgetracker/tracker_main.py
+++ b/ForgeTracker/forgetracker/tracker_main.py
@@ -1608,18 +1608,19 @@ class RootRestController(BaseController):
@expose('json:')
def perform_import(self, doc=None, options=None, **post_data):
- require_access(c.project, 'admin')
- if c.api_token.get_capability('import') != [c.project.neighborhood.name, c.project.shortname]:
- log.error('Import capability is not enabled for %s', c.project.shortname)
- raise exc.HTTPForbidden(detail='Import is not allowed')
+ with h.notifications_disabled(c.project):
+ require_access(c.project, 'admin')
+ if c.api_token.get_capability('import') != [c.project.neighborhood.name, c.project.shortname]:
+ log.error('Import capability is not enabled for %s', c.project.shortname)
+ raise exc.HTTPForbidden(detail='Import is not allowed')
- migrator = ImportSupport()
- try:
- status = migrator.perform_import(doc, options, **post_data)
- return status
- except Exception, e:
- log.exception(e)
- return dict(status=False, errors=[str(e)])
+ migrator = ImportSupport()
+ try:
+ status = migrator.perform_import(doc, options, **post_data)
+ return status
+ except Exception, e:
+ log.exception(e)
+ return dict(status=False, errors=[str(e)])
@expose('json:')
def search(self, q=None, limit=100, page=0, sort=None, **kw):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/675495e3/ForgeWiki/forgewiki/wiki_main.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/wiki_main.py b/ForgeWiki/forgewiki/wiki_main.py
index 816db78..e059038 100644
--- a/ForgeWiki/forgewiki/wiki_main.py
+++ b/ForgeWiki/forgewiki/wiki_main.py
@@ -730,17 +730,18 @@ class PageRestController(BaseController):
return self.page.__json__()
def _update_page(self, title, **post_data):
- if not self.page:
- require_access(c.app, 'create')
- self.page = WM.Page.upsert(title)
- self.page.viewable_by = ['all']
- else:
- require_access(self.page, 'edit')
- self.page.text = post_data['text']
- if 'labels' in post_data:
- self.page.labels = post_data['labels'].split(',')
- self.page.commit()
- return {}
+ with h.notifications_disabled(c.project):
+ if not self.page:
+ require_access(c.app, 'create')
+ self.page = WM.Page.upsert(title)
+ self.page.viewable_by = ['all']
+ else:
+ require_access(self.page, 'edit')
+ self.page.text = post_data['text']
+ if 'labels' in post_data:
+ self.page.labels = post_data['labels'].split(',')
+ self.page.commit()
+ return {}
class WikiAdminController(DefaultAdminController):
[6/6] git commit: [#6506] Added TracWikiImporter to
requirements-sf.txt
Posted by jo...@apache.org.
[#6506] Added TracWikiImporter to requirements-sf.txt
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3e67aa66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3e67aa66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3e67aa66
Branch: refs/heads/master
Commit: 3e67aa667a132c162719e394a5963261dfb1345d
Parents: 0875863
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Fri Aug 16 00:27:49 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Fri Aug 16 00:27:49 2013 +0000
----------------------------------------------------------------------
requirements-sf.txt | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3e67aa66/requirements-sf.txt
----------------------------------------------------------------------
diff --git a/requirements-sf.txt b/requirements-sf.txt
index b774faf..6d006ec 100644
--- a/requirements-sf.txt
+++ b/requirements-sf.txt
@@ -20,6 +20,7 @@ wsgipreload==1.2
pyzmq==2.1.7
html2text==3.200.3dev-20121112
PyMollom==0.1
+TracWikiImporter=0.1.0
# use version built from https://github.com/johnsca/GitPython/commits/tv/6000
# for unmerged fixes for [#5411], [#6000], and [#6078]
[5/6] git commit: [#6506] Fixed unicode and attribute errors
Posted by jo...@apache.org.
[#6506] Fixed unicode and attribute errors
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/08758635
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/08758635
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/08758635
Branch: refs/heads/master
Commit: 0875863598e47a5d1990f3ae6b0d8edd04ddaa6c
Parents: 37aacca
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Thu Aug 15 20:55:35 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Thu Aug 15 20:55:35 2013 +0000
----------------------------------------------------------------------
ForgeTracker/forgetracker/import_support.py | 12 ++++++------
ForgeTracker/forgetracker/scripts/import_tracker.py | 2 +-
.../forgewiki/scripts/wiki_from_trac/extractors.py | 6 +++---
3 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/08758635/ForgeTracker/forgetracker/import_support.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/import_support.py b/ForgeTracker/forgetracker/import_support.py
index e22c50e..9637ebc 100644
--- a/ForgeTracker/forgetracker/import_support.py
+++ b/ForgeTracker/forgetracker/import_support.py
@@ -199,10 +199,10 @@ class ImportSupport(object):
new_f, conv = transform
remapped[new_f] = conv(v)
- description = self.link_processing(remapped['description'])
+ description = h.really_unicode(self.link_processing(remapped['description']))
if ticket_dict['submitter'] and not remapped['reported_by_id']:
- description = 'Originally created by: {0}\n\n{1}'.format(
- ticket_dict['submitter'], description)
+ description = u'Originally created by: {0}\n\n{1}'.format(
+ h.really_unicode(ticket_dict['submitter']), description)
remapped['description'] = description
ticket_num = ticket_dict['id']
@@ -273,10 +273,10 @@ class ImportSupport(object):
def make_comment(self, thread, comment_dict):
ts = self.parse_date(comment_dict['date'])
author_id = self.get_user_id(comment_dict['submitter'])
- text = self.link_processing(comment_dict['comment'])
+ text = h.really_unicode(self.link_processing(comment_dict['comment']))
if not author_id and comment_dict['submitter']:
- text = 'Originally posted by: {0}\n\n{1}'.format(
- comment_dict['submitter'], text)
+ text = u'Originally posted by: {0}\n\n{1}'.format(
+ h.really_unicode(comment_dict['submitter']), text)
comment = thread.post(text=text, timestamp=ts)
comment.author_id = author_id
comment.import_id = c.api_token.api_key
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/08758635/ForgeTracker/forgetracker/scripts/import_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/scripts/import_tracker.py b/ForgeTracker/forgetracker/scripts/import_tracker.py
index 32b4d1c..c60a775 100644
--- a/ForgeTracker/forgetracker/scripts/import_tracker.py
+++ b/ForgeTracker/forgetracker/scripts/import_tracker.py
@@ -60,7 +60,7 @@ def import_tracker(cli, project, tool, import_options, doc_txt,
doc_import['trackers']['default'] = {}
doc_import['trackers']['default']['artifacts'] = [ticket_in]
res = cli.call(url, doc=json.dumps(doc_import), options=json.dumps(import_options))
- assert res['status'] and not res['errors']
+ assert res['status'] and not res['errors'], res['errors']
if validate:
if res['warnings']:
print "Ticket id %s warnings: %s" % (ticket_in['id'], res['warnings'])
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/08758635/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py b/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
index af07f50..6415bbf 100644
--- a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
+++ b/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
@@ -138,7 +138,7 @@ class WikiExporter(object):
url = urljoin(self.base_url, self.PAGE_LIST_URL)
self.log('Fetching list of pages from %s' % url)
r = self.fetch(url)
- html = BeautifulSoup(r.content)
+ html = BeautifulSoup(r)
pages = html.find('div', attrs=self.CONTENT_DIV_ATTRS) \
.find('ul').findAll('li')
pages = [page.find('a').text
@@ -165,14 +165,14 @@ class WikiExporter(object):
url = self.url(self.PAGE_URL % title)
self.log('Fetching page %s' % url)
r = self.fetch(url)
- html = BeautifulSoup(r.content)
+ html = BeautifulSoup(r)
return html.find('div', attrs=self.CONTENT_DIV_ATTRS)
def _get_page_regex(self, title):
url = self.url(self.PAGE_URL % title, 'txt')
self.log('Fetching page %s' % url)
r = self.fetch(url)
- return r.content
+ return r
def convert_title(self, title):
title = self.RENAME_PAGES.get(title, title)
[3/6] git commit: [#6506] Minor mocking cleanup
Posted by jo...@apache.org.
[#6506] Minor mocking cleanup
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3f6bf33c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3f6bf33c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3f6bf33c
Branch: refs/heads/master
Commit: 3f6bf33c4177e1e9889950383830f9374214c3eb
Parents: 490c83e
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Thu Aug 15 16:31:10 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Thu Aug 15 16:31:10 2013 +0000
----------------------------------------------------------------------
Allura/allura/lib/decorators.py | 6 +-----
Allura/allura/tests/test_decorators.py | 7 +++----
2 files changed, 4 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3f6bf33c/Allura/allura/lib/decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index 137152e..8604a92 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -32,10 +32,6 @@ from pylons import tmpl_context as c
from allura.lib import helpers as h
-def _get_model():
- from allura import model as M
- return M
-
def task(*args, **kw):
"""Decorator that adds a ``.post()`` function to the decorated callable.
@@ -62,7 +58,7 @@ def task(*args, **kw):
cm = (h.notifications_disabled if project and
kw.get('notifications_disabled') else h.null_contextmanager)
with cm(project):
- M = _get_model()
+ from allura import model as M
return M.MonQTask.post(func, args, kwargs, delay=delay)
# if decorating a class, have to make it a staticmethod
# or it gets a spurious cls argument
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3f6bf33c/Allura/allura/tests/test_decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_decorators.py b/Allura/allura/tests/test_decorators.py
index 893f327..41ba631 100644
--- a/Allura/allura/tests/test_decorators.py
+++ b/Allura/allura/tests/test_decorators.py
@@ -20,8 +20,8 @@ class TestTask(TestCase):
self.assertTrue(hasattr(func, 'post'))
@patch('allura.lib.decorators.c')
- @patch('allura.lib.decorators._get_model')
- def test_post(self, c, _get_model):
+ @patch('allura.model.MonQTask')
+ def test_post(self, c, MonQTask):
@task(disable_notifications=True)
def func(s, foo=None, **kw):
pass
@@ -34,6 +34,5 @@ class TestTask(TestCase):
self.assertEqual(f, func)
c.project.notifications_disabled = False
- M = _get_model.return_value
- M.MonQTask.post.side_effect = mock_post
+ MonQTask.post.side_effect = mock_post
func.post('test', foo=2, delay=1)
[4/6] git commit: [#6506] Shortened Trac Ticket Importer tool_label
Posted by jo...@apache.org.
[#6506] Shortened Trac Ticket Importer tool_label
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/37aacca9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/37aacca9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/37aacca9
Branch: refs/heads/master
Commit: 37aacca9a59e94c4cdfb440c5e96c0f6fe202314
Parents: 3f6bf33
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Thu Aug 15 20:22:44 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Thu Aug 15 20:22:44 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/trac/tickets.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/37aacca9/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index 00bca2a..ef099ca 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -83,7 +83,7 @@ class TracTicketImporter(ToolImporter):
target_app = ForgeTrackerApp
source = 'Trac'
controller = TracTicketImportController
- tool_label = 'Trac Ticket Importer'
+ tool_label = 'Tickets'
tool_description = 'Import your tickets from Trac'
def import_tool(self, project, user, project_name=None, mount_point=None,
[2/6] git commit: [#6506] Add custom User-Agent and auto-retries for
project export requests
Posted by jo...@apache.org.
[#6506] Add custom User-Agent and auto-retries for project export requests
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/490c83e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/490c83e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/490c83e3
Branch: refs/heads/master
Commit: 490c83e38af601ddeed4b4f7ca3ed1a7e5721ed0
Parents: 675495e
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Aug 13 21:37:31 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Thu Aug 15 16:09:31 2013 +0000
----------------------------------------------------------------------
Allura/allura/scripts/trac_export.py | 17 +++++++++++------
ForgeImporters/forgeimporters/base.py | 16 ++++++++++++++++
ForgeImporters/forgeimporters/google/__init__.py | 8 ++++----
.../forgeimporters/tests/google/test_extractor.py | 7 ++++---
ForgeImporters/forgeimporters/tests/test_base.py | 13 +++++++++++++
ForgeTracker/forgetracker/import_support.py | 10 +++++++---
.../forgewiki/scripts/wiki_from_trac/extractors.py | 17 ++++++++++-------
7 files changed, 65 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/490c83e3/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
index d53afbc..4d908bc 100644
--- a/Allura/allura/scripts/trac_export.py
+++ b/Allura/allura/scripts/trac_export.py
@@ -18,7 +18,6 @@
# under the License.
import logging
-import socket
import sys
import csv
import urlparse
@@ -28,14 +27,20 @@ import time
import re
from optparse import OptionParser
from itertools import islice
-from datetime import datetime
import feedparser
from BeautifulSoup import BeautifulSoup, NavigableString
import dateutil.parser
import pytz
-from allura.lib import helpers as h
+try:
+ from forgeimporters.base import ProjectExtractor
+ urlopen = ProjectExtractor.urlopen
+except ImportError:
+ try:
+ from allura.lib.helpers import urlopen
+ except ImportError:
+ from urllib2 import urlopen
log = logging.getLogger(__name__)
@@ -124,7 +129,7 @@ class TracExport(object):
def csvopen(self, url):
self.log_url(url)
- f = h.urlopen(url)
+ f = urlopen(url)
# Trac doesn't throw 403 error, just shows normal 200 HTML page
# telling that access denied. So, we'll emulate 403 ourselves.
# TODO: currently, any non-csv result treated as 403.
@@ -146,7 +151,7 @@ class TracExport(object):
from html2text import html2text
url = self.full_url(self.TICKET_URL % id, 'rss')
self.log_url(url)
- d = feedparser.parse(h.urlopen(url))
+ d = feedparser.parse(urlopen(url))
res = []
for comment in d['entries']:
c = {}
@@ -163,7 +168,7 @@ class TracExport(object):
# Scrape HTML to get ticket attachments
url = self.full_url(self.ATTACHMENT_LIST_URL % id)
self.log_url(url)
- f = h.urlopen(url)
+ f = urlopen(url)
soup = BeautifulSoup(f)
attach = soup.find('div', id='attachments')
list = []
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/490c83e3/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index e1af4f7..ee34ab3 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -16,6 +16,7 @@
# under the License.
import logging
+import urllib2
from pkg_resources import iter_entry_points
@@ -28,6 +29,7 @@ from allura.lib.decorators import require_post
from allura.lib.decorators import task
from allura.lib.security import require_access
from allura.lib.plugin import ProjectRegistrationProvider
+from allura.lib import helpers as h
from allura.lib import exceptions
from paste.deploy.converters import aslist
@@ -57,6 +59,20 @@ def import_tool(importer_name, project_name=None, mount_point=None, mount_label=
mount_point=mount_point, mount_label=mount_label, **kw)
+class ProjectExtractor(object):
+ """Base class for project extractors.
+
+ Subclasses should use :meth:`urlopen` to make HTTP requests, as it provides
+ a custom User-Agent and automatically retries timed-out requests.
+
+ """
+ @staticmethod
+ def urlopen(url, retries=3, codes=(408,), **kw):
+ req = urllib2.Request(url, **kw)
+ req.add_header('User-Agent', 'Allura Data Importer (http://sf.net/p/allura)')
+ return h.urlopen(req, retries=retries, codes=codes)
+
+
class ProjectImporter(BaseController):
"""
Base class for project importers.
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/490c83e3/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index a307bcd..a12389b 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -17,7 +17,6 @@
import re
import urllib
-import urllib2
from urlparse import urlparse, urljoin
from collections import defaultdict
try:
@@ -29,11 +28,12 @@ import logging
from BeautifulSoup import BeautifulSoup
from allura import model as M
+from forgeimporters.base import ProjectExtractor
log = logging.getLogger(__name__)
-class GoogleCodeProjectExtractor(object):
+class GoogleCodeProjectExtractor(ProjectExtractor):
BASE_URL = 'http://code.google.com'
RE_REPO_TYPE = re.compile(r'(svn|hg|git)')
@@ -82,7 +82,7 @@ class GoogleCodeProjectExtractor(object):
self.url = (self.get_page_url(page_name_or_url) if page_name_or_url in
self.PAGE_MAP else page_name_or_url)
self.page = self._page_cache[page_name_or_url] = \
- BeautifulSoup(urllib2.urlopen(self.url))
+ BeautifulSoup(self.urlopen(self.url))
return self.page
def get_page_url(self, page_name):
@@ -103,7 +103,7 @@ class GoogleCodeProjectExtractor(object):
if icon_url == self.DEFAULT_ICON:
return
icon_name = urllib.unquote(urlparse(icon_url).path).split('/')[-1]
- fp_ish = urllib2.urlopen(icon_url)
+ fp_ish = self.urlopen(icon_url)
fp = StringIO(fp_ish.read())
M.ProjectFile.save_image(
icon_name, fp,
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/490c83e3/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index b4e64c0..9b6db45 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -19,12 +19,13 @@ from unittest import TestCase
import mock
-from ... import google
+from forgeimporters import google
+from forgeimporters import base
class TestGoogleCodeProjectExtractor(TestCase):
def setUp(self):
- self._p_urlopen = mock.patch.object(google.urllib2, 'urlopen')
+ self._p_urlopen = mock.patch.object(base.ProjectExtractor, 'urlopen')
self._p_soup = mock.patch.object(google, 'BeautifulSoup')
self.urlopen = self._p_urlopen.start()
self.soup = self._p_soup.start()
@@ -105,7 +106,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
def _make_extractor(self, html):
from BeautifulSoup import BeautifulSoup
- with mock.patch.object(google, 'urllib2'):
+ with mock.patch.object(base.ProjectExtractor, 'urlopen'):
extractor = google.GoogleCodeProjectExtractor(self.project, 'my-project')
extractor.page = BeautifulSoup(html)
extractor.get_page = lambda pagename: extractor.page
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/490c83e3/ForgeImporters/forgeimporters/tests/test_base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/test_base.py b/ForgeImporters/forgeimporters/tests/test_base.py
index 68be24d..57ed227 100644
--- a/ForgeImporters/forgeimporters/tests/test_base.py
+++ b/ForgeImporters/forgeimporters/tests/test_base.py
@@ -23,6 +23,19 @@ import mock
from .. import base
+class TestProjectExtractor(TestCase):
+ @mock.patch('forgeimporters.base.h.urlopen')
+ @mock.patch('forgeimporters.base.urllib2.Request')
+ def test_urlopen(self, Request, urlopen):
+ r = base.ProjectExtractor.urlopen('myurl', data='foo')
+ Request.assert_called_once_with('myurl', data='foo')
+ req = Request.return_value
+ req.add_header.assert_called_once_with(
+ 'User-Agent', 'Allura Data Importer (http://sf.net/p/allura)')
+ urlopen.assert_called_once_with(req, retries=3, codes=(408,))
+ self.assertEqual(r, urlopen.return_value)
+
+
@mock.patch.object(base.ToolImporter, 'by_name')
@mock.patch.object(base, 'c')
def test_import_tool(c, by_name):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/490c83e3/ForgeTracker/forgetracker/import_support.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/import_support.py b/ForgeTracker/forgetracker/import_support.py
index 04d453a..e22c50e 100644
--- a/ForgeTracker/forgetracker/import_support.py
+++ b/ForgeTracker/forgetracker/import_support.py
@@ -34,6 +34,12 @@ from allura import model as M
# Local imports
from forgetracker import model as TM
+try:
+ from forgeimporters.base import ProjectExtractor
+ urlopen = ProjectExtractor.urlopen
+except ImportError:
+ urlopen = h.urlopen
+
log = logging.getLogger(__name__)
class ImportException(Exception):
@@ -276,15 +282,13 @@ class ImportSupport(object):
comment.import_id = c.api_token.api_key
def make_attachment(self, org_ticket_id, ticket_id, att_dict):
- import urllib2
if att_dict['size'] > self.ATTACHMENT_SIZE_LIMIT:
self.errors.append('Ticket #%s: Attachment %s (@ %s) is too large, skipping' %
(org_ticket_id, att_dict['filename'], att_dict['url']))
return
- f = urllib2.urlopen(att_dict['url'])
+ f = urlopen(att_dict['url'])
TM.TicketAttachment.save_attachment(att_dict['filename'], ResettableStream(f),
artifact_id=ticket_id)
- f.close()
#
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/490c83e3/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py b/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
index 7f111d7..af07f50 100644
--- a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
+++ b/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
@@ -24,15 +24,18 @@ from urllib import quote, unquote
from urlparse import urljoin, urlsplit
try:
- import requests
-except:
- # Ignore this import if the requests package is not installed
- pass
+ from forgeimporters.base import ProjectExtractor
+ urlopen = ProjectExtractor.urlopen
+except ImportError:
+ try:
+ from allura.lib.helpers import urlopen
+ except ImportError:
+ from urllib2 import urlopen
try:
# Ignore this import if the html2text package is not installed
import html2text
-except:
+except ImportError:
pass
from BeautifulSoup import BeautifulSoup
@@ -128,8 +131,8 @@ class WikiExporter(object):
glue = '&' if '?' in suburl else '?'
return url + glue + 'format=' + type
- def fetch(self, url, **kwargs):
- return requests.get(url, **kwargs)
+ def fetch(self, url):
+ return urlopen(url)
def page_list(self):
url = urljoin(self.base_url, self.PAGE_LIST_URL)