You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/08/07 19:02:47 UTC
[01/13] git commit: [#6480] TracExport bug fixes
Updated Branches:
refs/heads/cj/6464 08e4f2a9c -> cc16a2642 (forced update)
[#6480] TracExport bug fixes
- Pass in options explicitly instead of attempting to read from a
non-existent global object.
- Improve logging.
- Fix infinite loop bug.
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/35cc655a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/35cc655a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/35cc655a
Branch: refs/heads/cj/6464
Commit: 35cc655a2913532a6471488d82a6452365e08e4d
Parents: 280aff0
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Aug 6 18:50:46 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:23 2013 +0000
----------------------------------------------------------------------
Allura/allura/scripts/trac_export.py | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/35cc655a/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
index aeb14ea..bce548e 100644
--- a/Allura/allura/scripts/trac_export.py
+++ b/Allura/allura/scripts/trac_export.py
@@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.
-
+import logging
import sys
import csv
import urlparse
@@ -35,6 +35,8 @@ from BeautifulSoup import BeautifulSoup, NavigableString
import dateutil.parser
import pytz
+log = logging.getLogger(__name__)
+
def parse_options():
optparser = OptionParser(usage=''' %prog <Trac URL>
@@ -66,7 +68,7 @@ class TracExport(object):
'owner': 'assigned_to',
}
- def __init__(self, base_url, start_id=1):
+ def __init__(self, base_url, start_id=1, verbose=False, do_attachments=True):
"""start_id - start with at least that ticket number (actual returned
ticket may have higher id if we don't have access to exact
one).
@@ -78,6 +80,9 @@ class TracExport(object):
self.ticket_map = {}
self.start_id = start_id
self.page = (start_id - 1) / self.PAGE_SIZE + 1
+ self.verbose = verbose
+ self.do_attachments = do_attachments
+ self.exhausted = False
self.ticket_queue = self.next_ticket_ids()
def remap_fields(self, dict):
@@ -98,9 +103,9 @@ class TracExport(object):
glue = '&' if '?' in suburl else '?'
return url + glue + 'format=' + type
- @staticmethod
- def log_url(url):
- if options.verbose:
+ def log_url(self, url):
+ log.info(url)
+ if self.verbose:
print >>sys.stderr, url
@classmethod
@@ -198,7 +203,7 @@ class TracExport(object):
'''
t = self.parse_ticket_body(id)
t['comments'] = self.parse_ticket_comments(id)
- if options.do_attachments:
+ if self.do_attachments:
atts = self.parse_ticket_attachments(id)
if atts:
t['attachments'] = atts
@@ -230,6 +235,9 @@ class TracExport(object):
res.append((id, extra))
self.page += 1
+ if len(res) < self.PAGE_SIZE:
+ self.exhausted = True
+
return res
def __iter__(self):
@@ -238,7 +246,7 @@ class TracExport(object):
def next(self):
while True:
# queue empty, try to fetch more
- if len(self.ticket_queue) == 0:
+ if len(self.ticket_queue) == 0 and not self.exhausted:
self.ticket_queue = self.next_ticket_ids()
# there aren't any more, we're really done
if len(self.ticket_queue) == 0:
@@ -258,7 +266,8 @@ class DateJSONEncoder(json.JSONEncoder):
def main():
options, args = parse_options()
- ex = TracExport(args[0], start_id=options.start_id)
+ ex = TracExport(args[0], start_id=options.start_id,
+ verbose=options.verbose, do_attachments=options.do_attachments)
# Implement iterator sequence limiting using islice()
doc = [t for t in islice(ex, options.limit)]
[08/13] git commit: [#6480] Importer bug fixes
Posted by jo...@apache.org.
[#6480] Importer bug fixes
- Normalize trac urls
- Flush new app configs and related objects before importing artifacts
- Add options needed for WikiExporter
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/75539774
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/75539774
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/75539774
Branch: refs/heads/cj/6464
Commit: 755397743b8956fe2342eede82d4163b6fd92a3d
Parents: 35cc655
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Aug 6 18:57:37 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:24 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/base.py | 4 +++-
ForgeImporters/forgeimporters/google/code.py | 1 +
ForgeImporters/forgeimporters/trac/tickets.py | 9 +++++++--
ForgeImporters/forgeimporters/trac/wiki.py | 10 +++++++++-
ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py | 4 ++++
5 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/75539774/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 5652474..7ad720c 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -52,7 +52,9 @@ class ProjectImportForm(schema.Schema):
@task
def import_tool(importer_name, project_name, mount_point=None, mount_label=None, **kw):
importer = ToolImporter.by_name(importer_name)
- importer.import_tool(c.project, mount_point, mount_label, **kw)
+ importer.import_tool(project=c.project, user=c.user,
+ mount_point=mount_point,
+ mount_label=mount_label, **kw)
class ProjectImporter(BaseController):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/75539774/ForgeImporters/forgeimporters/google/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/code.py b/ForgeImporters/forgeimporters/google/code.py
index ef7f800..4457b55 100644
--- a/ForgeImporters/forgeimporters/google/code.py
+++ b/ForgeImporters/forgeimporters/google/code.py
@@ -101,6 +101,7 @@ class GoogleRepoImporter(ToolImporter):
tool_description = 'Import your SVN, Git, or Hg repo from Google Code'
def import_tool(self, project, project_name, mount_point=None, mount_label=None):
+ def import_tool(self, project, project_name, mount_point=None, mount_label=None, **kw):
""" Import a Google Code repo into a new SVN, Git, or Hg Allura tool.
"""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/75539774/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index 969dfd2..0f13649 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -24,6 +24,7 @@ import json
import formencode as fe
from formencode import validators as fev
+from ming.orm import session
from pylons import tmpl_context as c
from pylons import app_globals as g
from tg import (
@@ -84,21 +85,25 @@ class TracTicketImporter(ToolImporter):
tool_description = 'Import your tickets from Trac'
def import_tool(self, project=None, mount_point=None, mount_label=None,
- trac_url=None, user=None):
+ trac_url=None, user=None, **kw):
""" Import Trac tickets into a new Allura Tracker tool.
"""
+ trac_url = trac_url.rstrip('/') + '/'
mount_point = mount_point or 'tickets'
app = project.install_app(
'Tickets',
mount_point=mount_point,
mount_label=mount_label or 'Tickets',
)
- export = TracExport(trac_url)
+ session(app.config).flush(app.config)
+ session(app.globals).flush(app.globals)
+ export = [ticket for ticket in TracExport(trac_url)]
export_string = json.dumps(export, cls=DateJSONEncoder)
api_ticket = ApiTicket(user_id=user._id,
capabilities={"import": ["Projects", project.shortname]},
expires=datetime.utcnow() + timedelta(minutes=60))
+ session(api_ticket).flush(api_ticket)
cli = AlluraImportApiClient(config['base_url'], api_ticket.api_key,
api_ticket.secret_key, False)
import_tracker(cli, project.shortname, mount_point, {},
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/75539774/ForgeImporters/forgeimporters/trac/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/wiki.py b/ForgeImporters/forgeimporters/trac/wiki.py
index 2417863..e7ead86 100644
--- a/ForgeImporters/forgeimporters/trac/wiki.py
+++ b/ForgeImporters/forgeimporters/trac/wiki.py
@@ -25,6 +25,7 @@ import tempfile
import formencode as fe
from formencode import validators as fev
+from ming.orm import session
from pylons import tmpl_context as c
from pylons import app_globals as g
from tg import (
@@ -83,25 +84,32 @@ class TracWikiImporter(ToolImporter):
tool_description = 'Import your wiki from Trac'
def import_tool(self, project=None, mount_point=None, mount_label=None,
- trac_url=None, user=None):
+ trac_url=None, user=None, **kw):
""" Import Trac wiki into a new Allura Wiki tool.
"""
+ trac_url = trac_url.rstrip('/') + '/'
mount_point = mount_point or 'wiki'
app = project.install_app(
'Wiki',
mount_point=mount_point,
mount_label=mount_label or 'Wiki',
)
+ session(app.config).flush(app.config)
api_ticket = ApiTicket(user_id=user._id,
capabilities={"import": ["Projects", project.shortname]},
expires=datetime.utcnow() + timedelta(minutes=60))
+ session(api_ticket).flush(api_ticket)
options = argparse.Namespace()
options.api_key = api_ticket.api_key
options.secret_key = api_ticket.secret_key
options.project = project.shortname
options.wiki = mount_point
options.base_url = config['base_url']
+ options.verbose = False
+ options.converter = 'html2text'
+ options.import_opts = []
+ options.user_map_file = None
with tempfile.NamedTemporaryFile() as f:
WikiExporter(trac_url, options).export(f)
f.flush()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/75539774/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py b/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
index 0038dd9..7f111d7 100644
--- a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
+++ b/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import logging
import re
import sys
import json
@@ -36,6 +37,8 @@ except:
from BeautifulSoup import BeautifulSoup
+log = logging.getLogger(__name__)
+
class WikiExporter(object):
@@ -114,6 +117,7 @@ class WikiExporter(object):
out.write('\n')
def log(self, msg):
+ log.info(msg)
if self.options.verbose:
print >>sys.stderr, msg
[09/13] git commit: [#6480] Moved Trac wiki importer to separate
package
Posted by jo...@apache.org.
[#6480] Moved Trac wiki importer to separate package
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/04cd1ed2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/04cd1ed2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/04cd1ed2
Branch: refs/heads/cj/6464
Commit: 04cd1ed28c73fd25791d4fa6a0ad32a5fec2f157
Parents: 85c29b9
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Wed Aug 7 14:27:44 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:24 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/google/project.py | 2 +-
.../trac/templates/wiki/index.html | 42 -------
.../forgeimporters/trac/tests/test_wiki.py | 104 -----------------
ForgeImporters/forgeimporters/trac/wiki.py | 117 -------------------
ForgeImporters/setup.py | 1 -
5 files changed, 1 insertion(+), 265 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/04cd1ed2/ForgeImporters/forgeimporters/google/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/project.py b/ForgeImporters/forgeimporters/google/project.py
index dda41ea..e69ac9e 100644
--- a/ForgeImporters/forgeimporters/google/project.py
+++ b/ForgeImporters/forgeimporters/google/project.py
@@ -53,7 +53,7 @@ class GoogleCodeProjectImporter(base.ProjectImporter):
def after_project_create(self, project, **kw):
project.set_tool_data('google-code', project_name=project.name)
- tasks.import_project_info.post()
+ tasks.import_project_info.post(project.name)
@with_trailing_slash
@expose(index_template)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/04cd1ed2/ForgeImporters/forgeimporters/trac/templates/wiki/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/wiki/index.html b/ForgeImporters/forgeimporters/trac/templates/wiki/index.html
deleted file mode 100644
index 6083b9c..0000000
--- a/ForgeImporters/forgeimporters/trac/templates/wiki/index.html
+++ /dev/null
@@ -1,42 +0,0 @@
-{#-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--#}
-{% extends g.theme.master %}
-
-{% block title %}
-{{c.project.name}} / Import Trac Wiki
-{% endblock %}
-
-{% block header %}
-Import wiki from Trac
-{% endblock %}
-
-{% block content %}
-<form action="create" method="post" class="pad">
- <label for="trac_url">URL of the Trac instance</label>
- <input name="trac_url" />
-
- <label for="mount_label">Label</label>
- <input name="mount_label" value="Source" />
-
- <label for="mount_point">Mount Point</label>
- <input name="mount_point" value="source" />
-
- <input type="submit" />
-</form>
-{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/04cd1ed2/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py b/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
deleted file mode 100644
index 738e49b..0000000
--- a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
+++ /dev/null
@@ -1,104 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from unittest import TestCase
-from mock import Mock, patch
-
-from allura.tests import TestController
-from allura.tests.decorators import with_wiki
-
-from forgeimporters.trac.wiki import (
- TracWikiImporter,
- TracWikiImportController,
- )
-
-
-class TestWikiTicketImporter(TestCase):
- @patch('forgeimporters.trac.wiki.session')
- @patch('forgeimporters.trac.wiki.tempfile.NamedTemporaryFile')
- @patch('forgeimporters.trac.wiki.g')
- @patch('forgeimporters.trac.wiki.WikiFromTrac')
- @patch('forgeimporters.trac.wiki.load_data')
- @patch('forgeimporters.trac.wiki.argparse.Namespace')
- @patch('forgeimporters.trac.wiki.WikiExporter')
- @patch('forgeimporters.trac.wiki.ApiTicket')
- @patch('forgeimporters.trac.wiki.datetime')
- def test_import_tool(self, dt, ApiTicket, WikiExporter, Namespace,
- load_data, WikiFromTrac, g, NamedTemporaryFile, session):
- from datetime import datetime, timedelta
- now = datetime.utcnow()
- dt.utcnow.return_value = now
- export_file = NamedTemporaryFile.return_value.__enter__.return_value
- export_file.name = '/my/file'
-
- importer = TracWikiImporter()
- app = Mock(name='ForgeWikiApp')
- project = Mock(name='Project', shortname='myproject')
- project.install_app.return_value = app
- user = Mock(name='User', _id='id')
- res = importer.import_tool(project, user,
- mount_point='pages',
- mount_label='Pages',
- trac_url='http://example.com/trac/url')
- self.assertEqual(res, app)
- project.install_app.assert_called_once_with(
- 'Wiki', mount_point='pages', mount_label='Pages')
- ApiTicket.assert_called_once_with(
- user_id=user._id,
- capabilities={"import": ["Projects", "myproject"]},
- expires=now + timedelta(minutes=60))
- WikiExporter.assert_called_once_with('http://example.com/trac/url/',
- Namespace.return_value)
- WikiExporter.return_value.export.assert_called_once_with(export_file)
- load_data.assert_called_once_with('/my/file',
- WikiFromTrac.parser.return_value, Namespace.return_value)
- g.post_event.assert_called_once_with('project_updated')
-
-
-class TestTracWikiImportController(TestController, TestCase):
- def setUp(self):
- """Mount Trac import controller on the Wiki admin controller"""
- super(self.__class__, self).setUp()
- from forgewiki.wiki_main import WikiAdminController
- WikiAdminController._importer = TracWikiImportController()
-
- @with_wiki
- def test_index(self):
- r = self.app.get('/p/test/admin/wiki/_importer/')
- self.assertIsNotNone(r.html.find(attrs=dict(name="trac_url")))
- self.assertIsNotNone(r.html.find(attrs=dict(name="mount_label")))
- self.assertIsNotNone(r.html.find(attrs=dict(name="mount_point")))
-
- @with_wiki
- @patch('forgeimporters.trac.wiki.TracWikiImporter')
- def test_create(self, importer):
- from allura import model as M
- importer = importer.return_value
- importer.import_tool.return_value = Mock()
- importer.import_tool.return_value.url.return_value = '/p/test/mymount'
- params = dict(trac_url='http://example.com/trac/url',
- mount_label='mylabel',
- mount_point='mymount',
- )
- r = self.app.post('/p/test/admin/wiki/_importer/create', params,
- status=302)
- project = M.Project.query.get(shortname='test')
- self.assertEqual(r.location, 'http://localhost/p/test/mymount')
- self.assertEqual(project._id, importer.import_tool.call_args[0][0]._id)
- self.assertEqual(u'mymount', importer.import_tool.call_args[1]['mount_point'])
- self.assertEqual(u'mylabel', importer.import_tool.call_args[1]['mount_label'])
- self.assertEqual(u'http://example.com/trac/url', importer.import_tool.call_args[1]['trac_url'])
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/04cd1ed2/ForgeImporters/forgeimporters/trac/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/wiki.py b/ForgeImporters/forgeimporters/trac/wiki.py
deleted file mode 100644
index 300b476..0000000
--- a/ForgeImporters/forgeimporters/trac/wiki.py
+++ /dev/null
@@ -1,117 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import argparse
-from datetime import (
- datetime,
- timedelta,
- )
-import tempfile
-
-import formencode as fe
-from formencode import validators as fev
-
-from ming.orm import session
-from pylons import tmpl_context as c
-from pylons import app_globals as g
-from tg import (
- config,
- expose,
- redirect,
- validate,
- )
-from tg.decorators import (
- with_trailing_slash,
- without_trailing_slash,
- )
-
-from allura.controllers import BaseController
-from allura.lib.decorators import require_post
-from allura.model import ApiTicket
-
-from forgeimporters.base import ToolImporter
-
-from forgewiki.scripts.wiki_from_trac.extractors import WikiExporter
-from forgewiki.scripts.wiki_from_trac.loaders import load_data
-from forgewiki.scripts.wiki_from_trac.wiki_from_trac import WikiFromTrac
-from forgewiki.wiki_main import ForgeWikiApp
-
-
-class TracWikiImportSchema(fe.Schema):
- trac_url = fev.URL(not_empty=True)
- mount_point = fev.UnicodeString()
- mount_label = fev.UnicodeString()
-
-
-class TracWikiImportController(BaseController):
- @with_trailing_slash
- @expose('jinja:forgeimporters.trac:templates/wiki/index.html')
- def index(self, **kw):
- return {}
-
- @without_trailing_slash
- @expose()
- @require_post()
- @validate(TracWikiImportSchema(), error_handler=index)
- def create(self, trac_url, mount_point, mount_label, **kw):
- app = TracWikiImporter().import_tool(c.project, c.user,
- mount_point=mount_point,
- mount_label=mount_label,
- trac_url=trac_url)
- redirect(app.url())
-
-
-class TracWikiImporter(ToolImporter):
- target_app = ForgeWikiApp
- source = 'Trac'
- controller = TracWikiImportController
- tool_label = 'Trac Wiki Importer'
- tool_description = 'Import your wiki from Trac'
-
- def import_tool(self, project, user, project_name=None, mount_point=None,
- mount_label=None, trac_url=None, **kw):
- """ Import Trac wiki into a new Allura Wiki tool.
-
- """
- trac_url = trac_url.rstrip('/') + '/'
- mount_point = mount_point or 'wiki'
- app = project.install_app(
- 'Wiki',
- mount_point=mount_point,
- mount_label=mount_label or 'Wiki',
- )
- session(app.config).flush(app.config)
- api_ticket = ApiTicket(user_id=user._id,
- capabilities={"import": ["Projects", project.shortname]},
- expires=datetime.utcnow() + timedelta(minutes=60))
- session(api_ticket).flush(api_ticket)
- options = argparse.Namespace()
- options.api_key = api_ticket.api_key
- options.secret_key = api_ticket.secret_key
- options.project = project.shortname
- options.wiki = mount_point
- options.base_url = config['base_url']
- options.verbose = False
- options.converter = 'html2text'
- options.import_opts = []
- options.user_map_file = None
- with tempfile.NamedTemporaryFile() as f:
- WikiExporter(trac_url, options).export(f)
- f.flush()
- load_data(f.name, WikiFromTrac.parser(), options)
- g.post_event('project_updated')
- return app
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/04cd1ed2/ForgeImporters/setup.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py
index c4776ba..b19fd17 100644
--- a/ForgeImporters/setup.py
+++ b/ForgeImporters/setup.py
@@ -41,5 +41,4 @@ setup(name='ForgeImporters',
google-code-tracker = forgeimporters.google.tracker:GoogleCodeTrackerImporter
google-code-repo = forgeimporters.google.code:GoogleRepoImporter
trac-tickets = forgeimporters.trac.tickets:TracTicketImporter
- trac-wiki = forgeimporters.trac.wiki:TracWikiImporter
""",)
[03/13] git commit: [#6480] Add trac ticket importer plugin
Posted by jo...@apache.org.
[#6480] Add trac ticket importer plugin
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/512ee4b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/512ee4b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/512ee4b1
Branch: refs/heads/cj/6464
Commit: 512ee4b11f6fe09be909d539ff51f5b14824954c
Parents: e3663fb
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Fri Aug 2 17:00:29 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:23 2013 +0000
----------------------------------------------------------------------
Allura/allura/scripts/trac_export.py | 280 +++++++++++++++++++
ForgeImporters/forgeimporters/trac/__init__.py | 17 ++
.../trac/templates/tickets/index.html | 42 +++
ForgeImporters/forgeimporters/trac/tickets.py | 107 +++++++
.../forgetracker/scripts/import_tracker.py | 18 +-
scripts/trac_export.py | 257 +----------------
6 files changed, 458 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/512ee4b1/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
new file mode 100644
index 0000000..aeb14ea
--- /dev/null
+++ b/Allura/allura/scripts/trac_export.py
@@ -0,0 +1,280 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import sys
+import csv
+import urlparse
+import urllib2
+import json
+import time
+import re
+from optparse import OptionParser
+from itertools import islice
+from datetime import datetime
+
+import feedparser
+from html2text import html2text
+from BeautifulSoup import BeautifulSoup, NavigableString
+import dateutil.parser
+import pytz
+
+
+def parse_options():
+ optparser = OptionParser(usage=''' %prog <Trac URL>
+
+Export ticket data from a Trac instance''')
+ optparser.add_option('-o', '--out-file', dest='out_filename', help='Write to file (default stdout)')
+ optparser.add_option('--no-attachments', dest='do_attachments', action='store_false', default=True, help='Export attachment info')
+ optparser.add_option('--only-tickets', dest='only_tickets', action='store_true', help='Export only ticket list')
+ optparser.add_option('--start', dest='start_id', type='int', default=1, help='Start with given ticket numer (or next accessible)')
+ optparser.add_option('--limit', dest='limit', type='int', default=None, help='Limit number of tickets')
+ optparser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Verbose operation')
+ options, args = optparser.parse_args()
+ if len(args) != 1:
+ optparser.error("Wrong number of arguments.")
+ return options, args
+
+
+class TracExport(object):
+
+ PAGE_SIZE = 100
+ TICKET_URL = 'ticket/%d'
+ QUERY_MAX_ID_URL = 'query?col=id&order=id&desc=1&max=2'
+ QUERY_BY_PAGE_URL = 'query?col=id&col=time&col=changetime&order=id&max=' + str(PAGE_SIZE)+ '&page=%d'
+ ATTACHMENT_LIST_URL = 'attachment/ticket/%d/'
+ ATTACHMENT_URL = 'raw-attachment/ticket/%d/%s'
+
+ FIELD_MAP = {
+ 'reporter': 'submitter',
+ 'owner': 'assigned_to',
+ }
+
+ def __init__(self, base_url, start_id=1):
+ """start_id - start with at least that ticket number (actual returned
+ ticket may have higher id if we don't have access to exact
+ one).
+ """
+ self.base_url = base_url.rstrip('/') + '/'
+ # Contains additional info for a ticket which cannot
+ # be get with single-ticket export (create/mod times is
+ # and example).
+ self.ticket_map = {}
+ self.start_id = start_id
+ self.page = (start_id - 1) / self.PAGE_SIZE + 1
+ self.ticket_queue = self.next_ticket_ids()
+
+ def remap_fields(self, dict):
+ "Remap fields to adhere to standard taxonomy."
+ out = {}
+ for k, v in dict.iteritems():
+ out[self.FIELD_MAP.get(k, k)] = v
+
+ out['id'] = int(out['id'])
+ if 'private' in out:
+ out['private'] = bool(int(out['private']))
+ return out
+
+ def full_url(self, suburl, type=None):
+ url = urlparse.urljoin(self.base_url, suburl)
+ if type is None:
+ return url
+ glue = '&' if '?' in suburl else '?'
+ return url + glue + 'format=' + type
+
+ @staticmethod
+ def log_url(url):
+ if options.verbose:
+ print >>sys.stderr, url
+
+ @classmethod
+ def trac2z_date(cls, s):
+ d = dateutil.parser.parse(s)
+ d = d.astimezone(pytz.UTC)
+ return d.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+ @staticmethod
+ def match_pattern(regexp, string):
+ m = re.match(regexp, string)
+ assert m
+ return m.group(1)
+
+ def csvopen(self, url):
+ self.log_url(url)
+ f = urllib2.urlopen(url)
+ # Trac doesn't throw 403 error, just shows normal 200 HTML page
+ # telling that access denied. So, we'll emulate 403 ourselves.
+ # TODO: currently, any non-csv result treated as 403.
+ if not f.info()['Content-Type'].startswith('text/csv'):
+ raise urllib2.HTTPError(url, 403, 'Forbidden - emulated', f.info(), f)
+ return f
+
+ def parse_ticket_body(self, id):
+ # Use CSV export to get ticket fields
+ url = self.full_url(self.TICKET_URL % id, 'csv')
+ f = self.csvopen(url)
+ reader = csv.DictReader(f)
+ ticket_fields = reader.next()
+ ticket_fields['class'] = 'ARTIFACT'
+ return self.remap_fields(ticket_fields)
+
+ def parse_ticket_comments(self, id):
+ # Use RSS export to get ticket comments
+ url = self.full_url(self.TICKET_URL % id, 'rss')
+ self.log_url(url)
+ d = feedparser.parse(url)
+ res = []
+ for comment in d['entries']:
+ c = {}
+ c['submitter'] = comment.author
+ c['date'] = comment.updated_parsed
+ c['comment'] = html2text(comment.summary)
+ c['class'] = 'COMMENT'
+ res.append(c)
+ return res
+
+ def parse_ticket_attachments(self, id):
+ SIZE_PATTERN = r'(\d+) bytes'
+ TIMESTAMP_PATTERN = r'(.+) in Timeline'
+ # Scrape HTML to get ticket attachments
+ url = self.full_url(self.ATTACHMENT_LIST_URL % id)
+ self.log_url(url)
+ f = urllib2.urlopen(url)
+ soup = BeautifulSoup(f)
+ attach = soup.find('div', id='attachments')
+ list = []
+ while attach:
+ attach = attach.findNext('dt')
+ if not attach:
+ break
+ d = {}
+ d['filename'] = attach.a['href'].rsplit('/', 1)[1]
+ d['url'] = self.full_url(self.ATTACHMENT_URL % (id, d['filename']))
+ size_s = attach.span['title']
+ d['size'] = int(self.match_pattern(SIZE_PATTERN, size_s))
+ timestamp_s = attach.find('a', {'class': 'timeline'})['title']
+ d['date'] = self.trac2z_date(self.match_pattern(TIMESTAMP_PATTERN, timestamp_s))
+ d['by'] = attach.find(text=re.compile('added by')).nextSibling.renderContents()
+ d['description'] = ''
+ # Skip whitespace
+ while attach.nextSibling and type(attach.nextSibling) is NavigableString:
+ attach = attach.nextSibling
+ # if there's a description, there will be a <dd> element, other immediately next <dt>
+ if attach.nextSibling and attach.nextSibling.name == 'dd':
+ desc_el = attach.nextSibling
+ if desc_el:
+ # TODO: Convert to Allura link syntax as needed
+ d['description'] = ''.join(desc_el.findAll(text=True)).strip()
+ list.append(d)
+ return list
+
+ def get_max_ticket_id(self):
+ url = self.full_url(self.QUERY_MAX_ID_URL, 'csv')
+ f = self.csvopen(url)
+ reader = csv.DictReader(f)
+ fields = reader.next()
+ print fields
+ return int(fields['id'])
+
+ def get_ticket(self, id, extra={}):
+ '''Get ticket with given id
+ extra: extra fields to add to ticket (parsed elsewhere)
+ '''
+ t = self.parse_ticket_body(id)
+ t['comments'] = self.parse_ticket_comments(id)
+ if options.do_attachments:
+ atts = self.parse_ticket_attachments(id)
+ if atts:
+ t['attachments'] = atts
+ t.update(extra)
+ return t
+
+ def next_ticket_ids(self):
+ 'Go thru ticket list and collect available ticket ids.'
+ # We could just do CSV export, which by default dumps entire list
+ # Alas, for many busy servers with long ticket list, it will just
+ # time out. So, let's paginate it instead.
+ res = []
+
+ url = self.full_url(self.QUERY_BY_PAGE_URL % self.page, 'csv')
+ try:
+ f = self.csvopen(url)
+ except urllib2.HTTPError, e:
+ if 'emulated' in e.msg:
+ body = e.fp.read()
+ if 'beyond the number of pages in the query' in body or 'Log in with a SourceForge account' in body:
+ raise StopIteration
+ raise
+ reader = csv.reader(f)
+ cols = reader.next()
+ for r in reader:
+ if r and r[0].isdigit():
+ id = int(r[0])
+ extra = {'date': self.trac2z_date(r[1]), 'date_updated': self.trac2z_date(r[2])}
+ res.append((id, extra))
+ self.page += 1
+
+ return res
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ while True:
+ # queue empty, try to fetch more
+ if len(self.ticket_queue) == 0:
+ self.ticket_queue = self.next_ticket_ids()
+ # there aren't any more, we're really done
+ if len(self.ticket_queue) == 0:
+ raise StopIteration
+ id, extra = self.ticket_queue.pop(0)
+ if id >= self.start_id:
+ break
+ return self.get_ticket(id, extra)
+
+
+class DateJSONEncoder(json.JSONEncoder):
+ def default(self, obj):
+ if isinstance(obj, time.struct_time):
+ return time.strftime('%Y-%m-%dT%H:%M:%SZ', obj)
+ return json.JSONEncoder.default(self, obj)
+
+
+def main():
+ options, args = parse_options()
+ ex = TracExport(args[0], start_id=options.start_id)
+ # Implement iterator sequence limiting using islice()
+ doc = [t for t in islice(ex, options.limit)]
+
+ if not options.only_tickets:
+ doc = {
+ 'class': 'PROJECT',
+ 'trackers': {'default': {'artifacts': doc}}
+ }
+
+ out_file = sys.stdout
+ if options.out_filename:
+ out_file = open(options.out_filename, 'w')
+ out_file.write(json.dumps(doc, cls=DateJSONEncoder, indent=2, sort_keys=True))
+ # It's bad habit not to terminate lines
+ out_file.write('\n')
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/512ee4b1/ForgeImporters/forgeimporters/trac/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/__init__.py b/ForgeImporters/forgeimporters/trac/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/512ee4b1/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/tickets/index.html b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
new file mode 100644
index 0000000..eaf9aac
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/templates/tickets/index.html
@@ -0,0 +1,42 @@
+{#-
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
+{% extends g.theme.master %}
+
+{% block title %}
+{{c.project.name}} / Import Trac Tickets
+{% endblock %}
+
+{% block header %}
+Import tickets from Trac
+{% endblock %}
+
+{% block content %}
+<form action="create" method="post" class="pad">
+ <label for="trac_url">URL of the Trac instance</label>
+ <input name="trac_url" />
+
+ <label for="mount_label">Label</label>
+ <input name="mount_label" value="Source" />
+
+ <label for="mount_point">Mount Point</label>
+ <input name="mount_point" value="source" />
+
+ <input type="submit" />
+</form>
+{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/512ee4b1/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
new file mode 100644
index 0000000..cc31741
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from datetime import (
+ datetime,
+ timedelta,
+ )
+import json
+
+import formencode as fe
+from formencode import validators as fev
+
+from pylons import tmpl_context as c
+from pylons import app_globals as g
+from tg import (
+ config,
+ expose,
+ redirect,
+ validate,
+ )
+from tg.decorators import (
+ with_trailing_slash,
+ without_trailing_slash,
+ )
+
+from allura.controllers import BaseController
+from allura.lib.decorators import require_post
+from allura.lib.import_api import AlluraImportApiClient
+from allura.model import ApiTicket
+from allura.scripts.trac_export import (
+ TracExport,
+ DateJSONEncoder,
+ )
+
+from forgeimporters.base import ToolImporter
+from forgetracker.tracker_main import ForgeTrackerApp
+from forgetracker.script.import_tracker import import_tracker
+
+
+class TracTicketImportSchema(fe.Schema):
+ trac_url = fev.URL(not_empty=True)
+ mount_point = fev.UnicodeString()
+ mount_label = fev.UnicodeString()
+
+
+class TracTicketImportController(BaseController):
+ @with_trailing_slash
+ @expose('jinja:forgeimporters.trac:templates/tickets/index.html')
+ def index(self, **kw):
+ return {}
+
+ @without_trailing_slash
+ @expose()
+ @require_post()
+ @validate(TracTicketImportSchema(), error_handler=index)
+ def create(self, trac_url, mount_point, mount_label, **kw):
+ app = TracTicketImporter.import_tool(c.project,
+ mount_point=mount_point,
+ mount_label=mount_label,
+ trac_url=trac_url,
+ user=c.user)
+ redirect(app.url())
+
+
+class TracTicketImporter(ToolImporter):
+ target_app = ForgeTrackerApp
+ source = 'Trac'
+ controller = TracTicketImportController
+ tool_label = 'Trac Ticket Importer'
+ tool_description = 'Import your tickets from Trac'
+
+ def import_tool(self, project=None, mount_point=None, mount_label=None,
+ trac_url=None, user=None):
+ """ Import Trac tickets into a new Allura Tracker tool.
+
+ """
+ mount_point = mount_point or 'tickets'
+ app = project.install_app(
+ 'Tickets',
+ mount_point=mount_point,
+ mount_label=mount_label or 'Tickets',
+ )
+ export = TracExport(trac_url)
+ export_string = json.dumps(export, cls=DateJSONEncoder)
+ api_ticket = ApiTicket(user_id=user._id,
+ capabilities={"import": ["Projects", project.shortname]},
+ expires=datetime.utcnow() + timedelta(minutes=60))
+ cli = AlluraImportApiClient(config['base_url'], api_ticket.api_key,
+ api_ticket.secret_key, False)
+ import_tracker(cli, project.shortname, mount_point, {},
+ export_string, validate=False)
+ g.post_event('project_updated')
+ return app
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/512ee4b1/ForgeTracker/forgetracker/scripts/import_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeTracker/forgetracker/scripts/import_tracker.py b/ForgeTracker/forgetracker/scripts/import_tracker.py
index 506e771..32b4d1c 100644
--- a/ForgeTracker/forgetracker/scripts/import_tracker.py
+++ b/ForgeTracker/forgetracker/scripts/import_tracker.py
@@ -25,7 +25,8 @@ from allura.lib.import_api import AlluraImportApiClient
log = logging.getLogger(__name__)
-def import_tracker(cli, project, tool, import_options, options, doc_txt, validate=True, verbose=False):
+def import_tracker(cli, project, tool, import_options, doc_txt,
+ validate=True, verbose=False, cont=False):
url = '/rest/p/' + project + '/' + tool
if validate:
url += '/validate_import'
@@ -33,8 +34,8 @@ def import_tracker(cli, project, tool, import_options, options, doc_txt, validat
url += '/perform_import'
existing_map = {}
- if options.cont:
- existing_tickets = cli.call('/rest/p/' + options.project + '/' + options.tracker + '/')['tickets']
+ if cont:
+ existing_tickets = cli.call('/rest/p/' + project + '/' + tool + '/')['tickets']
for t in existing_tickets:
existing_map[t['ticket_num']] = t['summary']
@@ -46,12 +47,12 @@ def import_tracker(cli, project, tool, import_options, options, doc_txt, validat
else:
tickets_in = doc
- if options.verbose:
+ if verbose:
print "Processing %d tickets" % len(tickets_in)
for cnt, ticket_in in enumerate(tickets_in):
if ticket_in['id'] in existing_map:
- if options.verbose:
+ if verbose:
print 'Ticket id %d already exists, skipping' % ticket_in['id']
continue
doc_import={}
@@ -60,7 +61,7 @@ def import_tracker(cli, project, tool, import_options, options, doc_txt, validat
doc_import['trackers']['default']['artifacts'] = [ticket_in]
res = cli.call(url, doc=json.dumps(doc_import), options=json.dumps(import_options))
assert res['status'] and not res['errors']
- if options.validate:
+ if validate:
if res['warnings']:
print "Ticket id %s warnings: %s" % (ticket_in['id'], res['warnings'])
else:
@@ -93,9 +94,10 @@ class ImportTracker(ScriptTask):
import_options['user_map'] = user_map
cli = AlluraImportApiClient(options.base_url, options.api_key, options.secret_key, options.verbose)
doc_txt = open(options.file_data).read()
- import_tracker(cli, options.project, options.tracker, import_options, options, doc_txt,
+ import_tracker(cli, options.project, options.tracker, import_options, doc_txt,
validate=options.validate,
- verbose=options.verbose)
+ verbose=options.verbose,
+ cont=options.cont)
@classmethod
def parser(cls):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/512ee4b1/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/scripts/trac_export.py b/scripts/trac_export.py
index 002a1e8..ac90b17 100755
--- a/scripts/trac_export.py
+++ b/scripts/trac_export.py
@@ -17,259 +17,6 @@
# specific language governing permissions and limitations
# under the License.
-
-import sys
-import csv
-import urlparse
-import urllib2
-import json
-import time
-import re
-from optparse import OptionParser
-from itertools import islice
-from datetime import datetime
-
-import feedparser
-from html2text import html2text
-from BeautifulSoup import BeautifulSoup, NavigableString
-import dateutil.parser
-import pytz
-
-
-def parse_options():
- optparser = OptionParser(usage=''' %prog <Trac URL>
-
-Export ticket data from a Trac instance''')
- optparser.add_option('-o', '--out-file', dest='out_filename', help='Write to file (default stdout)')
- optparser.add_option('--no-attachments', dest='do_attachments', action='store_false', default=True, help='Export attachment info')
- optparser.add_option('--only-tickets', dest='only_tickets', action='store_true', help='Export only ticket list')
- optparser.add_option('--start', dest='start_id', type='int', default=1, help='Start with given ticket numer (or next accessible)')
- optparser.add_option('--limit', dest='limit', type='int', default=None, help='Limit number of tickets')
- optparser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Verbose operation')
- options, args = optparser.parse_args()
- if len(args) != 1:
- optparser.error("Wrong number of arguments.")
- return options, args
-
-
-class TracExport(object):
-
- PAGE_SIZE = 100
- TICKET_URL = 'ticket/%d'
- QUERY_MAX_ID_URL = 'query?col=id&order=id&desc=1&max=2'
- QUERY_BY_PAGE_URL = 'query?col=id&col=time&col=changetime&order=id&max=' + str(PAGE_SIZE)+ '&page=%d'
- ATTACHMENT_LIST_URL = 'attachment/ticket/%d/'
- ATTACHMENT_URL = 'raw-attachment/ticket/%d/%s'
-
- FIELD_MAP = {
- 'reporter': 'submitter',
- 'owner': 'assigned_to',
- }
-
- def __init__(self, base_url, start_id=1):
- """start_id - start with at least that ticket number (actual returned
- ticket may have higher id if we don't have access to exact
- one).
- """
- self.base_url = base_url.rstrip('/') + '/'
- # Contains additional info for a ticket which cannot
- # be get with single-ticket export (create/mod times is
- # and example).
- self.ticket_map = {}
- self.start_id = start_id
- self.page = (start_id - 1) / self.PAGE_SIZE + 1
- self.ticket_queue = self.next_ticket_ids()
-
- def remap_fields(self, dict):
- "Remap fields to adhere to standard taxonomy."
- out = {}
- for k, v in dict.iteritems():
- out[self.FIELD_MAP.get(k, k)] = v
-
- out['id'] = int(out['id'])
- if 'private' in out:
- out['private'] = bool(int(out['private']))
- return out
-
- def full_url(self, suburl, type=None):
- url = urlparse.urljoin(self.base_url, suburl)
- if type is None:
- return url
- glue = '&' if '?' in suburl else '?'
- return url + glue + 'format=' + type
-
- @staticmethod
- def log_url(url):
- if options.verbose:
- print >>sys.stderr, url
-
- @classmethod
- def trac2z_date(cls, s):
- d = dateutil.parser.parse(s)
- d = d.astimezone(pytz.UTC)
- return d.strftime("%Y-%m-%dT%H:%M:%SZ")
-
- @staticmethod
- def match_pattern(regexp, string):
- m = re.match(regexp, string)
- assert m
- return m.group(1)
-
- def csvopen(self, url):
- self.log_url(url)
- f = urllib2.urlopen(url)
- # Trac doesn't throw 403 error, just shows normal 200 HTML page
- # telling that access denied. So, we'll emulate 403 ourselves.
- # TODO: currently, any non-csv result treated as 403.
- if not f.info()['Content-Type'].startswith('text/csv'):
- raise urllib2.HTTPError(url, 403, 'Forbidden - emulated', f.info(), f)
- return f
-
- def parse_ticket_body(self, id):
- # Use CSV export to get ticket fields
- url = self.full_url(self.TICKET_URL % id, 'csv')
- f = self.csvopen(url)
- reader = csv.DictReader(f)
- ticket_fields = reader.next()
- ticket_fields['class'] = 'ARTIFACT'
- return self.remap_fields(ticket_fields)
-
- def parse_ticket_comments(self, id):
- # Use RSS export to get ticket comments
- url = self.full_url(self.TICKET_URL % id, 'rss')
- self.log_url(url)
- d = feedparser.parse(url)
- res = []
- for comment in d['entries']:
- c = {}
- c['submitter'] = comment.author
- c['date'] = comment.updated_parsed
- c['comment'] = html2text(comment.summary)
- c['class'] = 'COMMENT'
- res.append(c)
- return res
-
- def parse_ticket_attachments(self, id):
- SIZE_PATTERN = r'(\d+) bytes'
- TIMESTAMP_PATTERN = r'(.+) in Timeline'
- # Scrape HTML to get ticket attachments
- url = self.full_url(self.ATTACHMENT_LIST_URL % id)
- self.log_url(url)
- f = urllib2.urlopen(url)
- soup = BeautifulSoup(f)
- attach = soup.find('div', id='attachments')
- list = []
- while attach:
- attach = attach.findNext('dt')
- if not attach:
- break
- d = {}
- d['filename'] = attach.a['href'].rsplit('/', 1)[1]
- d['url'] = self.full_url(self.ATTACHMENT_URL % (id, d['filename']))
- size_s = attach.span['title']
- d['size'] = int(self.match_pattern(SIZE_PATTERN, size_s))
- timestamp_s = attach.find('a', {'class': 'timeline'})['title']
- d['date'] = self.trac2z_date(self.match_pattern(TIMESTAMP_PATTERN, timestamp_s))
- d['by'] = attach.find(text=re.compile('added by')).nextSibling.renderContents()
- d['description'] = ''
- # Skip whitespace
- while attach.nextSibling and type(attach.nextSibling) is NavigableString:
- attach = attach.nextSibling
- # if there's a description, there will be a <dd> element, other immediately next <dt>
- if attach.nextSibling and attach.nextSibling.name == 'dd':
- desc_el = attach.nextSibling
- if desc_el:
- # TODO: Convert to Allura link syntax as needed
- d['description'] = ''.join(desc_el.findAll(text=True)).strip()
- list.append(d)
- return list
-
- def get_max_ticket_id(self):
- url = self.full_url(self.QUERY_MAX_ID_URL, 'csv')
- f = self.csvopen(url)
- reader = csv.DictReader(f)
- fields = reader.next()
- print fields
- return int(fields['id'])
-
- def get_ticket(self, id, extra={}):
- '''Get ticket with given id
- extra: extra fields to add to ticket (parsed elsewhere)
- '''
- t = self.parse_ticket_body(id)
- t['comments'] = self.parse_ticket_comments(id)
- if options.do_attachments:
- atts = self.parse_ticket_attachments(id)
- if atts:
- t['attachments'] = atts
- t.update(extra)
- return t
-
- def next_ticket_ids(self):
- 'Go thru ticket list and collect available ticket ids.'
- # We could just do CSV export, which by default dumps entire list
- # Alas, for many busy servers with long ticket list, it will just
- # time out. So, let's paginate it instead.
- res = []
-
- url = self.full_url(self.QUERY_BY_PAGE_URL % self.page, 'csv')
- try:
- f = self.csvopen(url)
- except urllib2.HTTPError, e:
- if 'emulated' in e.msg:
- body = e.fp.read()
- if 'beyond the number of pages in the query' in body or 'Log in with a SourceForge account' in body:
- raise StopIteration
- raise
- reader = csv.reader(f)
- cols = reader.next()
- for r in reader:
- if r and r[0].isdigit():
- id = int(r[0])
- extra = {'date': self.trac2z_date(r[1]), 'date_updated': self.trac2z_date(r[2])}
- res.append((id, extra))
- self.page += 1
-
- return res
-
- def __iter__(self):
- return self
-
- def next(self):
- while True:
- # queue empty, try to fetch more
- if len(self.ticket_queue) == 0:
- self.ticket_queue = self.next_ticket_ids()
- # there aren't any more, we're really done
- if len(self.ticket_queue) == 0:
- raise StopIteration
- id, extra = self.ticket_queue.pop(0)
- if id >= self.start_id:
- break
- return self.get_ticket(id, extra)
-
-
-class DateJSONEncoder(json.JSONEncoder):
- def default(self, obj):
- if isinstance(obj, time.struct_time):
- return time.strftime('%Y-%m-%dT%H:%M:%SZ', obj)
- return json.JSONEncoder.default(self, obj)
-
if __name__ == '__main__':
- options, args = parse_options()
- ex = TracExport(args[0], start_id=options.start_id)
- # Implement iterator sequence limiting using islice()
- doc = [t for t in islice(ex, options.limit)]
-
- if not options.only_tickets:
- doc = {
- 'class': 'PROJECT',
- 'trackers': {'default': {'artifacts': doc}}
- }
-
- out_file = sys.stdout
- if options.out_filename:
- out_file = open(options.out_filename, 'w')
- out_file.write(json.dumps(doc, cls=DateJSONEncoder, indent=2, sort_keys=True))
- # It's bad habit not to terminate lines
- out_file.write('\n')
+ from allura.scripts.trac_export import main
+ main()
[11/13] git commit: [#6480] Initialize classes to prevent exc if no
href
Posted by jo...@apache.org.
[#6480] Initialize classes to prevent exc if no href
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/f398a06e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/f398a06e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/f398a06e
Branch: refs/heads/cj/6464
Commit: f398a06e1ca979922c366d30ad17060b53eb5988
Parents: 7553977
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Aug 6 19:23:16 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:24 2013 +0000
----------------------------------------------------------------------
Allura/allura/lib/markdown_extensions.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/f398a06e/Allura/allura/lib/markdown_extensions.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/markdown_extensions.py b/Allura/allura/lib/markdown_extensions.py
index 4879bf4..6e1004d 100644
--- a/Allura/allura/lib/markdown_extensions.py
+++ b/Allura/allura/lib/markdown_extensions.py
@@ -99,6 +99,7 @@ class ForgeLinkPattern(markdown.inlinepatterns.LinkPattern):
except IndexError:
title = None
+ classes = ''
if href:
if href == 'TOC':
return '[TOC]' # skip TOC
[10/13] git commit: [#6480] Add tests for Trac wiki importer
Posted by jo...@apache.org.
[#6480] Add tests for Trac wiki importer
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/2379488a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/2379488a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/2379488a
Branch: refs/heads/cj/6464
Commit: 2379488ac978b40c027f28720c92a97d9d54a585
Parents: 8a365f7
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Aug 6 21:59:58 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:24 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/trac/tests/test_tickets.py | 1 +
.../forgeimporters/trac/tests/test_wiki.py | 88 ++++++++++++++++++++
ForgeImporters/forgeimporters/trac/tickets.py | 2 +-
ForgeImporters/forgeimporters/trac/wiki.py | 2 +-
4 files changed, 91 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2379488a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 8b102a9..2cce886 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -84,6 +84,7 @@ class TestTracTicketImportController(TestController, TestCase):
@patch('forgeimporters.trac.tickets.TracTicketImporter')
def test_create(self, importer):
from allura import model as M
+ importer = importer.return_value
importer.import_tool.return_value = Mock()
importer.import_tool.return_value.url.return_value = '/p/test/mymount'
params = dict(trac_url='http://example.com/trac/url',
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2379488a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py b/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
index 77505f1..71cc8b5 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
@@ -15,3 +15,91 @@
# specific language governing permissions and limitations
# under the License.
+from unittest import TestCase
+from mock import Mock, patch
+
+from allura.tests import TestController
+from allura.tests.decorators import with_wiki
+
+from forgeimporters.trac.wiki import (
+ TracWikiImporter,
+ TracWikiImportController,
+ )
+
+
+class TestWikiTicketImporter(TestCase):
+ @patch('forgeimporters.trac.wiki.session')
+ @patch('forgeimporters.trac.wiki.tempfile.NamedTemporaryFile')
+ @patch('forgeimporters.trac.wiki.g')
+ @patch('forgeimporters.trac.wiki.WikiFromTrac')
+ @patch('forgeimporters.trac.wiki.load_data')
+ @patch('forgeimporters.trac.wiki.argparse.Namespace')
+ @patch('forgeimporters.trac.wiki.WikiExporter')
+ @patch('forgeimporters.trac.wiki.ApiTicket')
+ @patch('forgeimporters.trac.wiki.datetime')
+ def test_import_tool(self, dt, ApiTicket, WikiExporter, Namespace,
+ load_data, WikiFromTrac, g, NamedTemporaryFile, session):
+ from datetime import datetime, timedelta
+ now = datetime.utcnow()
+ dt.utcnow.return_value = now
+ export_file = NamedTemporaryFile.return_value.__enter__.return_value
+ export_file.name = '/my/file'
+
+ importer = TracWikiImporter()
+ app = Mock(name='ForgeWikiApp')
+ project = Mock(name='Project', shortname='myproject')
+ project.install_app.return_value = app
+ user = Mock(name='User', _id='id')
+ res = importer.import_tool(project=project,
+ mount_point='pages',
+ mount_label='Pages',
+ trac_url='http://example.com/trac/url',
+ user=user)
+ self.assertEqual(res, app)
+ project.install_app.assert_called_once_with(
+ 'Wiki', mount_point='pages', mount_label='Pages')
+ ApiTicket.assert_called_once_with(
+ user_id=user._id,
+ capabilities={"import": ["Projects", "myproject"]},
+ expires=now + timedelta(minutes=60))
+ WikiExporter.assert_called_once_with('http://example.com/trac/url/',
+ Namespace.return_value)
+ WikiExporter.return_value.export.assert_called_once_with(export_file)
+ load_data.assert_called_once_with('/my/file',
+ WikiFromTrac.parser.return_value, Namespace.return_value)
+ g.post_event.assert_called_once_with('project_updated')
+
+
+class TestTracWikiImportController(TestController, TestCase):
+ def setUp(self):
+ """Mount Trac import controller on the Wiki admin controller"""
+ super(self.__class__, self).setUp()
+ from forgewiki.wiki_main import WikiAdminController
+ WikiAdminController._importer = TracWikiImportController()
+
+ @with_wiki
+ def test_index(self):
+ r = self.app.get('/p/test/admin/wiki/_importer/')
+ self.assertIsNotNone(r.html.find(attrs=dict(name="trac_url")))
+ self.assertIsNotNone(r.html.find(attrs=dict(name="mount_label")))
+ self.assertIsNotNone(r.html.find(attrs=dict(name="mount_point")))
+
+ @with_wiki
+ @patch('forgeimporters.trac.wiki.TracWikiImporter')
+ def test_create(self, importer):
+ from allura import model as M
+ importer = importer.return_value
+ importer.import_tool.return_value = Mock()
+ importer.import_tool.return_value.url.return_value = '/p/test/mymount'
+ params = dict(trac_url='http://example.com/trac/url',
+ mount_label='mylabel',
+ mount_point='mymount',
+ )
+ r = self.app.post('/p/test/admin/wiki/_importer/create', params,
+ status=302)
+ project = M.Project.query.get(shortname='test')
+ self.assertEqual(r.location, 'http://localhost/p/test/mymount')
+ self.assertEqual(project._id, importer.import_tool.call_args[0][0]._id)
+ self.assertEqual(u'mymount', importer.import_tool.call_args[1]['mount_point'])
+ self.assertEqual(u'mylabel', importer.import_tool.call_args[1]['mount_label'])
+ self.assertEqual(u'http://example.com/trac/url', importer.import_tool.call_args[1]['trac_url'])
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2379488a/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index 0f13649..78d8d17 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -69,7 +69,7 @@ class TracTicketImportController(BaseController):
@require_post()
@validate(TracTicketImportSchema(), error_handler=index)
def create(self, trac_url, mount_point, mount_label, **kw):
- app = TracTicketImporter.import_tool(c.project,
+ app = TracTicketImporter().import_tool(c.project,
mount_point=mount_point,
mount_label=mount_label,
trac_url=trac_url,
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2379488a/ForgeImporters/forgeimporters/trac/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/wiki.py b/ForgeImporters/forgeimporters/trac/wiki.py
index e7ead86..00f0d48 100644
--- a/ForgeImporters/forgeimporters/trac/wiki.py
+++ b/ForgeImporters/forgeimporters/trac/wiki.py
@@ -68,7 +68,7 @@ class TracWikiImportController(BaseController):
@require_post()
@validate(TracWikiImportSchema(), error_handler=index)
def create(self, trac_url, mount_point, mount_label, **kw):
- app = TracWikiImporter.import_tool(c.project,
+ app = TracWikiImporter().import_tool(c.project,
mount_point=mount_point,
mount_label=mount_label,
trac_url=trac_url,
[05/13] git commit: [#6480] Fix tests to match refactors
Posted by jo...@apache.org.
[#6480] Fix tests to match refactors
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/1d5708d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/1d5708d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/1d5708d0
Branch: refs/heads/cj/6464
Commit: 1d5708d07dbac80cad787c99c5d4595559c52cd6
Parents: f398a06
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Aug 6 19:38:52 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:24 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/tests/google/test_tasks.py | 9 ---------
ForgeImporters/forgeimporters/tests/test_base.py | 14 +++++++++++++-
2 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/1d5708d0/ForgeImporters/forgeimporters/tests/google/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tasks.py b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
index 23da83f..dc7d936 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_tasks.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
@@ -31,12 +31,3 @@ def test_import_project_info(c, session, gpe):
gpe.return_value.get_icon.assert_called_once_with()
gpe.return_value.get_license.assert_called_once_with()
session.flush_all.assert_called_once_with()
-
-
-@mock.patch.object(tasks.ToolImporter, 'by_name')
-@mock.patch.object(tasks, 'c')
-def test_import_tool(c, by_name):
- c.project = mock.Mock(name='project')
- tasks.import_tool('importer_name', 'project_name', 'mount_point', 'mount_label')
- by_name.assert_called_once_with('importer_name')
- by_name.return_value.import_tool.assert_called_once_with(c.project, 'project_name', 'mount_point', 'mount_label')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/1d5708d0/ForgeImporters/forgeimporters/tests/test_base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/test_base.py b/ForgeImporters/forgeimporters/tests/test_base.py
index 1558db4..18b83ac 100644
--- a/ForgeImporters/forgeimporters/tests/test_base.py
+++ b/ForgeImporters/forgeimporters/tests/test_base.py
@@ -23,6 +23,18 @@ import mock
from .. import base
+@mock.patch.object(base.ToolImporter, 'by_name')
+@mock.patch.object(base, 'c')
+def test_import_tool(c, by_name):
+ c.project = mock.Mock(name='project')
+ c.user = mock.Mock(name='user')
+ base.import_tool('importer_name', 'project_name', 'mount_point', 'mount_label')
+ by_name.assert_called_once_with('importer_name')
+ by_name.return_value.import_tool.assert_called_once_with(c.project,
+ 'project_name', user=c.user, mount_point='mount_point',
+ mount_label='mount_label')
+
+
def ep(name, source=None, importer=None, **kw):
mep = mock.Mock(name='mock_ep', **kw)
mep.name = name
@@ -39,7 +51,7 @@ class TestProjectImporter(TestCase):
@mock.patch.object(base, 'iter_entry_points')
def test_tool_importers(self, iep):
eps = iep.return_value = [ep('ep1', 'foo'), ep('ep2', 'bar'), ep('ep3', 'foo')]
- pi = base.ProjectImporter()
+ pi = base.ProjectImporter(mock.Mock(name='neighborhood'))
pi.source = 'foo'
self.assertEqual(pi.tool_importers, {'ep1': eps[0].lv, 'ep3': eps[2].lv})
iep.assert_called_once_with('allura.importers')
[06/13] git commit: [#6480] Add tests for Trac ticket importer
Posted by jo...@apache.org.
[#6480] Add tests for Trac ticket importer
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/8a365f74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/8a365f74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/8a365f74
Branch: refs/heads/cj/6464
Commit: 8a365f74d9e8921e13a597fe34032b373ee49257
Parents: 1d5708d
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Tue Aug 6 21:12:10 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:24 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/google/tests/__init__.py | 17 ++++
.../forgeimporters/trac/tests/__init__.py | 17 ++++
.../forgeimporters/trac/tests/test_tickets.py | 100 +++++++++++++++++++
.../forgeimporters/trac/tests/test_wiki.py | 17 ++++
4 files changed, 151 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8a365f74/ForgeImporters/forgeimporters/google/tests/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tests/__init__.py b/ForgeImporters/forgeimporters/google/tests/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/google/tests/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8a365f74/ForgeImporters/forgeimporters/trac/tests/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/__init__.py b/ForgeImporters/forgeimporters/trac/tests/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/tests/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8a365f74/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
new file mode 100644
index 0000000..8b102a9
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+from mock import Mock, patch
+
+from allura.tests import TestController
+from allura.tests.decorators import with_tracker
+
+from forgeimporters.trac.tickets import (
+ TracTicketImporter,
+ TracTicketImportController,
+ )
+
+
+class TestTracTicketImporter(TestCase):
+ @patch('forgeimporters.trac.tickets.session')
+ @patch('forgeimporters.trac.tickets.g')
+ @patch('forgeimporters.trac.tickets.import_tracker')
+ @patch('forgeimporters.trac.tickets.AlluraImportApiClient')
+ @patch('forgeimporters.trac.tickets.datetime')
+ @patch('forgeimporters.trac.tickets.ApiTicket')
+ @patch('forgeimporters.trac.tickets.TracExport')
+ def test_import_tool(self, TracExport, ApiTicket, dt, ApiClient, import_tracker, g, session):
+ from datetime import datetime, timedelta
+ now = datetime.utcnow()
+ dt.utcnow.return_value = now
+
+ importer = TracTicketImporter()
+ app = Mock(name='ForgeTrackerApp')
+ project = Mock(name='Project', shortname='myproject')
+ project.install_app.return_value = app
+ user = Mock(name='User', _id='id')
+ res = importer.import_tool(project=project,
+ mount_point='bugs',
+ mount_label='Bugs',
+ trac_url='http://example.com/trac/url',
+ user=user)
+ self.assertEqual(res, app)
+ project.install_app.assert_called_once_with(
+ 'Tickets', mount_point='bugs', mount_label='Bugs')
+ TracExport.return_value = []
+ TracExport.assert_called_once_with('http://example.com/trac/url/')
+ ApiTicket.assert_called_once_with(
+ user_id=user._id,
+ capabilities={"import": ["Projects", "myproject"]},
+ expires=now + timedelta(minutes=60))
+ api_client = ApiClient.return_value
+ import_tracker.assert_called_once_with(
+ api_client, 'myproject', 'bugs', {}, '[]',
+ validate=False)
+ g.post_event.assert_called_once_with('project_updated')
+
+
+class TestTracTicketImportController(TestController, TestCase):
+ def setUp(self):
+ """Mount Trac import controller on the Tracker admin controller"""
+ super(TestTracTicketImportController, self).setUp()
+ from forgetracker.tracker_main import TrackerAdminController
+ TrackerAdminController._importer = TracTicketImportController()
+
+ @with_tracker
+ def test_index(self):
+ r = self.app.get('/p/test/admin/bugs/_importer/')
+ self.assertIsNotNone(r.html.find(attrs=dict(name="trac_url")))
+ self.assertIsNotNone(r.html.find(attrs=dict(name="mount_label")))
+ self.assertIsNotNone(r.html.find(attrs=dict(name="mount_point")))
+
+ @with_tracker
+ @patch('forgeimporters.trac.tickets.TracTicketImporter')
+ def test_create(self, importer):
+ from allura import model as M
+ importer.import_tool.return_value = Mock()
+ importer.import_tool.return_value.url.return_value = '/p/test/mymount'
+ params = dict(trac_url='http://example.com/trac/url',
+ mount_label='mylabel',
+ mount_point='mymount',
+ )
+ r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+ status=302)
+ project = M.Project.query.get(shortname='test')
+ self.assertEqual(r.location, 'http://localhost/p/test/mymount')
+ self.assertEqual(project._id, importer.import_tool.call_args[0][0]._id)
+ self.assertEqual(u'mymount', importer.import_tool.call_args[1]['mount_point'])
+ self.assertEqual(u'mylabel', importer.import_tool.call_args[1]['mount_label'])
+ self.assertEqual(u'http://example.com/trac/url', importer.import_tool.call_args[1]['trac_url'])
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8a365f74/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py b/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
[04/13] git commit: [#6480] Remove whitespace from shortname;
use autofocus
Posted by jo...@apache.org.
[#6480] Remove whitespace from shortname; use autofocus
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/280aff07
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/280aff07
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/280aff07
Branch: refs/heads/cj/6464
Commit: 280aff074794e8d7bd2a1924a93d3f6f6fdfbb32
Parents: 63fdf19
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Mon Aug 5 19:30:50 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:23 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/google/templates/project.html | 2 +-
ForgeImporters/forgeimporters/templates/project_base.html | 4 ++--
ForgeImporters/forgeimporters/trac/templates/project.html | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/280aff07/ForgeImporters/forgeimporters/google/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/templates/project.html b/ForgeImporters/forgeimporters/google/templates/project.html
index 172dcf5..a2887e7 100644
--- a/ForgeImporters/forgeimporters/google/templates/project.html
+++ b/ForgeImporters/forgeimporters/google/templates/project.html
@@ -23,7 +23,7 @@
<label>Google Project Name</label>
</div>
<div class="grid-10">
- <input id="project_name" name="project_name" value="{{c.form_values['project_name']}}"/>
+ <input id="project_name" name="project_name" value="{{c.form_values['project_name']}}" autofocus/>
<div id="project_name_error" class="error{% if not c.form_errors['project_name'] %} hidden{% endif %}">
{{c.form_errors['project_name']}}
</div>
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/280aff07/ForgeImporters/forgeimporters/templates/project_base.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/templates/project_base.html b/ForgeImporters/forgeimporters/templates/project_base.html
index d4db30e..6f1b683 100644
--- a/ForgeImporters/forgeimporters/templates/project_base.html
+++ b/ForgeImporters/forgeimporters/templates/project_base.html
@@ -47,7 +47,7 @@
function suggest_name() {
var $project_shortname = $('#project_shortname');
if (!manual) {
- $project_shortname.val($('#project_name').val());
+ $project_shortname.val($('#project_name').val().replace(/\s/g, '').toLowerCase());
}
$project_shortname.trigger('change');
}
@@ -72,7 +72,7 @@
}
$(function() {
- $('#project_name').focus().bind('change keyup', suggest_name);
+ $('#project_name').bind('change keyup', suggest_name);
$('#project_shortname').bind('change keyup', function(event) {
if (event.type == 'keyup') {
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/280aff07/ForgeImporters/forgeimporters/trac/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/project.html b/ForgeImporters/forgeimporters/trac/templates/project.html
index abcfb48..869e290 100644
--- a/ForgeImporters/forgeimporters/trac/templates/project.html
+++ b/ForgeImporters/forgeimporters/trac/templates/project.html
@@ -23,7 +23,7 @@
<label>Trac URL</label>
</div>
<div class="grid-10">
- <input id="trac_url" name="trac_url" value="{{c.form_values['trac_url']}}"/>
+ <input id="trac_url" name="trac_url" value="{{c.form_values['trac_url']}}" autofocus/>
<div id="trac_ur_errorl" class="error{% if not c.form_errors['trac_url'] %} hidden{% endif %}">
{{c.form_errors['trac_url']}}
</div>
[02/13] git commit: [#6480] Add trac project and wiki importers;
refactor bases
Posted by jo...@apache.org.
[#6480] Add trac project and wiki importers; refactor bases
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/63fdf19f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/63fdf19f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/63fdf19f
Branch: refs/heads/cj/6464
Commit: 63fdf19fda232d6b0774ee5c428f3fed1b80cd21
Parents: 512ee4b
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Fri Aug 2 22:01:08 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:23 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/base.py | 96 +++++++++++++++-
ForgeImporters/forgeimporters/google/project.py | 66 +++--------
ForgeImporters/forgeimporters/google/tasks.py | 6 -
.../google/templates/project.html | 62 -----------
.../forgeimporters/templates/project_base.html | 89 ++++++++++++++-
ForgeImporters/forgeimporters/trac/project.py | 63 +++++++++++
.../forgeimporters/trac/templates/project.html | 32 ++++++
.../trac/templates/wiki/index.html | 42 +++++++
ForgeImporters/forgeimporters/trac/tickets.py | 2 +-
ForgeImporters/forgeimporters/trac/wiki.py | 110 +++++++++++++++++++
ForgeImporters/setup.py | 3 +
11 files changed, 445 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 034a580..5652474 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -15,16 +15,46 @@
# specific language governing permissions and limitations
# under the License.
+import logging
+
from pkg_resources import iter_entry_points
-from tg import expose
+from tg import expose, validate, flash, redirect, config
+from tg.decorators import with_trailing_slash
+from pylons import tmpl_context as c
+from formencode import validators as fev, schema
+
+from allura.lib.decorators import require_post
+from allura.lib.decorators import task
+from allura.lib.security import require_access
+from allura.lib.widgets.forms import NeighborhoodProjectShortNameValidator
+from allura.lib import exceptions
+
from paste.deploy.converters import aslist
-from formencode import validators as fev
from ming.utils import LazyProperty
from allura.controllers import BaseController
+log = logging.getLogger(__name__)
+
+
+class ProjectImportForm(schema.Schema):
+ def __init__(self, source):
+ super(ProjectImportForm, self).__init__()
+ self.add_field('tools', ToolsValidator(source))
+
+ neighborhood = fev.PlainText(not_empty=True)
+ project_name = fev.UnicodeString(not_empty=True, max=40)
+ project_shortname = NeighborhoodProjectShortNameValidator()
+
+
+@task
+def import_tool(importer_name, project_name, mount_point=None, mount_label=None, **kw):
+ importer = ToolImporter.by_name(importer_name)
+ importer.import_tool(c.project, mount_point, mount_label, **kw)
+
+
class ProjectImporter(BaseController):
"""
Base class for project importers.
@@ -33,6 +63,14 @@ class ProjectImporter(BaseController):
:meth:`process()` views described below.
"""
source = None
+ process_validator = None
+ index_template = None
+
+ def __init__(self, neighborhood, *a, **kw):
+ self.neighborhood = neighborhood
+
+ def _check_security(self):
+ require_access(self.neighborhood, 'register')
@LazyProperty
def tool_importers(self):
@@ -47,6 +85,8 @@ class ProjectImporter(BaseController):
tools[ep.name] = epv()
return tools
+ @with_trailing_slash
+ @expose()
def index(self, **kw):
"""
Override and expose this view to present the project import form.
@@ -58,9 +98,12 @@ class ProjectImporter(BaseController):
This will list the available tool importers. Other project fields
(e.g., project_name) should go in the project_fields block.
"""
- raise NotImplemented
+ return {'importer': self, 'tg_template': self.index_template}
- def process(self, tools=None, **kw):
+ @require_post()
+ @expose()
+ @validate(process_validator, error_handler=index)
+ def process(self, **kw):
"""
Override and expose this to handle a project import.
@@ -68,7 +111,50 @@ class ProjectImporter(BaseController):
tools installed and redirect to the new project, presumably with a
message indicating that some data will not be available immediately.
"""
- raise NotImplemented
+ try:
+ c.project = self.neighborhood.register_project(kw['project_shortname'],
+ project_name=kw['project_name'])
+ except exceptions.ProjectOverlimitError:
+ flash("You have exceeded the maximum number of projects you are allowed to create", 'error')
+ redirect('.')
+ except exceptions.ProjectRatelimitError:
+ flash("Project creation rate limit exceeded. Please try again later.", 'error')
+ redirect('.')
+ except Exception:
+ log.error('error registering project: %s', kw['project_shortname'], exc_info=True)
+ flash('Internal Error. Please try again later.', 'error')
+ redirect('.')
+
+ self.after_project_create(c.project, **kw)
+ for importer_name in kw['tools']:
+ import_tool.post(importer_name, **kw)
+
+ flash('Welcome to the %s Project System! '
+ 'Your project data will be imported and should show up here shortly.' % config['site_name'])
+ redirect(c.project.script_name + 'admin/overview')
+
+ @expose('json:')
+ @validate(process_validator)
+ def check_names(self, **kw):
+ """
+ Ajax form validation.
+
+ """
+ return c.form_errors
+
+ def after_project_create(self, project, **kw):
+ """
+ Called after project is created.
+
+ Useful for doing extra processing on the project before individual
+ tool imports happen.
+
+ :param project: The newly created project.
+ :param \*\*kw: The keyword arguments that were posted to the controller
+ method that created the project.
+
+ """
+ pass
class ToolImporter(object):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/google/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/project.py b/ForgeImporters/forgeimporters/google/project.py
index 7416258..dda41ea 100644
--- a/ForgeImporters/forgeimporters/google/project.py
+++ b/ForgeImporters/forgeimporters/google/project.py
@@ -17,16 +17,12 @@
import logging
-from tg import expose, validate, flash, redirect, config
+from formencode import validators as fev
+
+from tg import expose, validate
from tg.decorators import with_trailing_slash
-from pylons import tmpl_context as c
-from formencode import validators as fev, schema
from allura.lib.decorators import require_post
-from allura.lib.widgets.forms import NeighborhoodProjectShortNameValidator
-from allura.lib.security import require_access
-from allura.lib import helpers as h
-from allura.lib import exceptions
from .. import base
from . import tasks
@@ -35,15 +31,12 @@ from . import tasks
log = logging.getLogger(__name__)
-class GoogleCodeProjectForm(schema.Schema):
- neighborhood = fev.PlainText(not_empty=True)
+class GoogleCodeProjectForm(base.ProjectImportForm):
project_name = fev.Regex(r'^[a-z0-9][a-z0-9-]{,61}$',
not_empty=True,
messages={
'invalid': 'Please use only letters, numbers, and dashes.',
})
- project_shortname = NeighborhoodProjectShortNameValidator()
- tools = base.ToolsValidator('Google Code')
class GoogleCodeProjectImporter(base.ProjectImporter):
@@ -55,50 +48,25 @@ class GoogleCodeProjectImporter(base.ProjectImporter):
import.
"""
source = 'Google Code'
+ process_validator = GoogleCodeProjectForm(source)
+ index_template = 'jinja:forgeimporters.google:templates/project.html'
- def __init__(self, neighborhood, *a, **kw):
- super(GoogleCodeProjectImporter, self).__init__(*a, **kw)
- self.neighborhood = neighborhood
-
- def _check_security(self):
- require_access(self.neighborhood, 'register')
+ def after_project_create(self, project, **kw):
+ project.set_tool_data('google-code', project_name=project.name)
+ tasks.import_project_info.post()
@with_trailing_slash
- @expose('jinja:forgeimporters.google:templates/project.html')
+ @expose(index_template)
def index(self, **kw):
- return {'importer': self}
+ return super(self.__class__, self).index(**kw)
@require_post()
@expose()
- @validate(GoogleCodeProjectForm(), error_handler=index)
- def process(self, project_name=None, project_shortname=None, tools=None, **kw):
- project_name = h.really_unicode(project_name).encode('utf-8')
- project_shortname = h.really_unicode(project_shortname).encode('utf-8').lower()
-
- try:
- c.project = self.neighborhood.register_project(project_shortname,
- project_name=project_name)
- except exceptions.ProjectOverlimitError:
- flash("You have exceeded the maximum number of projects you are allowed to create", 'error')
- redirect('.')
- except exceptions.ProjectRatelimitError:
- flash("Project creation rate limit exceeded. Please try again later.", 'error')
- redirect('.')
- except Exception as e:
- log.error('error registering project: %s', project_shortname, exc_info=True)
- flash('Internal Error. Please try again later.', 'error')
- redirect('.')
-
- c.project.set_tool_data('google-code', project_name=project_name)
- tasks.import_project_info.post(project_name)
- for importer_name in tools:
- tasks.import_tool.post(importer_name, project_name)
-
- flash('Welcome to the %s Project System! '
- 'Your project data will be imported and should show up here shortly.' % config['site_name'])
- redirect(c.project.script_name + 'admin/overview')
+ @validate(process_validator, error_handler=index)
+ def process(self, **kw):
+ return super(self.__class__, self).process(**kw)
@expose('json:')
- @validate(GoogleCodeProjectForm())
- def check_names(self, project_name=None, project_shortname=None, tools=None, **kw):
- return c.form_errors
+ @validate(process_validator)
+ def check_names(self, **kw):
+ return super(self.__class__, self).check_names(**kw)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/google/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tasks.py b/ForgeImporters/forgeimporters/google/tasks.py
index 3e6e74d..968d9a9 100644
--- a/ForgeImporters/forgeimporters/google/tasks.py
+++ b/ForgeImporters/forgeimporters/google/tasks.py
@@ -23,7 +23,6 @@ from ming.orm import ThreadLocalORMSession
from allura.lib.decorators import task
from . import GoogleCodeProjectExtractor
-from ..base import ToolImporter
@task
@@ -34,8 +33,3 @@ def import_project_info(project_name):
extractor.get_license()
ThreadLocalORMSession.flush_all()
g.post_event('project_updated')
-
-@task
-def import_tool(importer_name, project_name, mount_point=None, mount_label=None):
- importer = ToolImporter.by_name(importer_name)
- importer.import_tool(c.project, project_name, mount_point, mount_label)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/google/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/templates/project.html b/ForgeImporters/forgeimporters/google/templates/project.html
index 2cda0d0..172dcf5 100644
--- a/ForgeImporters/forgeimporters/google/templates/project.html
+++ b/ForgeImporters/forgeimporters/google/templates/project.html
@@ -18,68 +18,6 @@
-#}
{% extends 'forgeimporters:templates/project_base.html' %}
-{% block extra_css %}
- {{ super() }}
- <style type="text/css">
- #project-import-form #project-fields input {
- width: 88%;
- }
-
- .hidden { display: none; }
- </style>
-{% endblock %}
-
-{% block extra_js %}
- {{ super() }}
- <script type="text/javascript">
- var timers = {};
- function delay(callback, ms) {
- clearTimeout(timers[callback]);
- timers[callback] = setTimeout(callback, ms);
- }
-
- var manual = false;
- function suggest_name() {
- var $project_shortname = $('#project_shortname');
- if (!manual) {
- $project_shortname.val($('#project_name').val());
- }
- $project_shortname.trigger('change');
- }
-
- function check_names() {
- var data = {
- 'neighborhood': $('#neighborhood').val(),
- 'project_name': $('#project_name').val(),
- 'project_shortname': $('#project_shortname').val()
- };
- $.getJSON('check_names', data, function(result) {
- $('#project_name_error').addClass('hidden');
- $('#project_shortname_error').addClass('hidden');
- for(var field in result) {
- $('#'+field+'_error').text(result[field]).removeClass('hidden');
- }
- });
- }
-
- function update_url() {
- $('#url-fragment').text($('#project_shortname').val());
- }
-
- $(function() {
- $('#project_name').focus().bind('change keyup', suggest_name);
-
- $('#project_shortname').bind('change keyup', function(event) {
- if (event.type == 'keyup') {
- manual = true;
- }
- update_url();
- delay(check_names, 500);
- });
- });
- </script>
-{% endblock %}
-
{% block project_fields %}
<div class="grid-6">
<label>Google Project Name</label>
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/templates/project_base.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/templates/project_base.html b/ForgeImporters/forgeimporters/templates/project_base.html
index fc654f2..d4db30e 100644
--- a/ForgeImporters/forgeimporters/templates/project_base.html
+++ b/ForgeImporters/forgeimporters/templates/project_base.html
@@ -23,13 +23,97 @@
{% block title %}{{importer.source}} Project Importer{% endblock %}
{% block header %}{{importer.source}} Project Importer{% endblock %}
-{% block content %}
+{% block extra_css %}
+ {{ super() }}
+ <style type="text/css">
+ #project-import-form #project-fields input {
+ width: 88%;
+ }
+
+ .hidden { display: none; }
+ </style>
+{% endblock %}
+
+{% block extra_js %}
+ {{ super() }}
+ <script type="text/javascript">
+ var timers = {};
+ function delay(callback, ms) {
+ clearTimeout(timers[callback]);
+ timers[callback] = setTimeout(callback, ms);
+ }
+
+ var manual = false;
+ function suggest_name() {
+ var $project_shortname = $('#project_shortname');
+ if (!manual) {
+ $project_shortname.val($('#project_name').val());
+ }
+ $project_shortname.trigger('change');
+ }
+
+ function check_names() {
+ var data = {
+ 'neighborhood': $('#neighborhood').val(),
+ 'project_name': $('#project_name').val(),
+ 'project_shortname': $('#project_shortname').val()
+ };
+ $.getJSON('check_names', data, function(result) {
+ $('#project_name_error').addClass('hidden');
+ $('#project_shortname_error').addClass('hidden');
+ for(var field in result) {
+ $('#'+field+'_error').text(result[field]).removeClass('hidden');
+ }
+ });
+ }
+
+ function update_url() {
+ $('#url-fragment').text($('#project_shortname').val());
+ }
+ $(function() {
+ $('#project_name').focus().bind('change keyup', suggest_name);
+
+ $('#project_shortname').bind('change keyup', function(event) {
+ if (event.type == 'keyup') {
+ manual = true;
+ }
+ update_url();
+ delay(check_names, 500);
+ });
+ });
+ </script>
+{% endblock %}
+
+{% block content %}
<form id="project-import-form" method="POST" action="process">
<input type="hidden" id="neighborhood" name="neighborhood" value="{{importer.neighborhood.name}}"/>
<fieldset id="project-fields">
- {% block project_fields %}{% endblock %}
+ {% block project_fields %}
+ <div class="grid-6" style="clear:left">
+ <label>Project Name</label>
+ </div>
+ <div class="grid-10">
+ <input id="project_name" name="project_name" value="{{c.form_values['project_name']}}"/>
+ <div id="project_name_error" class="error{% if not c.form_errors['project_name'] %} hidden{% endif %}">
+ {{c.form_errors['project_name']}}
+ </div>
+ </div>
+
+ <div class="grid-6" style="clear:left">
+ <label>URL Name</label>
+ </div>
+ <div class="grid-10">
+ <input id="project_shortname" name="project_shortname" value="{{c.form_values['project_shortname']}}"/>
+ <div id="project_shortname_error" class="error{% if not c.form_errors['project_shortname'] %} hidden{% endif %}">
+ {{c.form_errors['project_shortname']}}
+ </div>
+ <div id="project-url">
+ http://{{request.environ['HTTP_HOST']}}{{importer.neighborhood.url()}}<span id="url-fragment">{{c.form_values['project_shortname']}}</span>
+ </div>
+ </div>
+ {% endblock %}
</fieldset>
<fieldset id="tool-fields">
@@ -50,5 +134,4 @@
<input type="submit" value="Import"/>
</form>
-
{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/trac/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/project.py b/ForgeImporters/forgeimporters/trac/project.py
new file mode 100644
index 0000000..66e8326
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/project.py
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from formencode import validators as fev
+
+from tg import expose, validate
+from tg.decorators import with_trailing_slash
+
+from allura.lib.decorators import require_post
+
+from .. import base
+
+
+log = logging.getLogger(__name__)
+
+
+class TracProjectForm(base.ProjectImportForm):
+ trac_url = fev.URL(not_empty=True)
+
+
+class TracProjectImporter(base.ProjectImporter):
+ """
+ Project importer for Trac.
+
+ """
+ source = 'Trac'
+ process_validator = TracProjectForm(source)
+ index_template = 'jinja:forgeimporters.trac:templates/project.html'
+
+ def after_project_create(self, project, **kw):
+ project.set_tool_data('trac', url=kw['trac_url'])
+
+ @with_trailing_slash
+ @expose(index_template)
+ def index(self, **kw):
+ return super(self.__class__, self).index(**kw)
+
+ @require_post()
+ @expose()
+ @validate(process_validator, error_handler=index)
+ def process(self, **kw):
+ return super(self.__class__, self).process(**kw)
+
+ @expose('json:')
+ @validate(process_validator)
+ def check_names(self, **kw):
+ return super(self.__class__, self).check_names(**kw)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/trac/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/project.html b/ForgeImporters/forgeimporters/trac/templates/project.html
new file mode 100644
index 0000000..abcfb48
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/templates/project.html
@@ -0,0 +1,32 @@
+{#-
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
+{% extends 'forgeimporters:templates/project_base.html' %}
+
+{% block project_fields %}
+ <div class="grid-6">
+ <label>Trac URL</label>
+ </div>
+ <div class="grid-10">
+ <input id="trac_url" name="trac_url" value="{{c.form_values['trac_url']}}"/>
+ <div id="trac_ur_errorl" class="error{% if not c.form_errors['trac_url'] %} hidden{% endif %}">
+ {{c.form_errors['trac_url']}}
+ </div>
+ </div>
+ {{ super() }}
+{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/trac/templates/wiki/index.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/templates/wiki/index.html b/ForgeImporters/forgeimporters/trac/templates/wiki/index.html
new file mode 100644
index 0000000..6083b9c
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/templates/wiki/index.html
@@ -0,0 +1,42 @@
+{#-
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
+{% extends g.theme.master %}
+
+{% block title %}
+{{c.project.name}} / Import Trac Wiki
+{% endblock %}
+
+{% block header %}
+Import wiki from Trac
+{% endblock %}
+
+{% block content %}
+<form action="create" method="post" class="pad">
+ <label for="trac_url">URL of the Trac instance</label>
+ <input name="trac_url" />
+
+ <label for="mount_label">Label</label>
+ <input name="mount_label" value="Source" />
+
+ <label for="mount_point">Mount Point</label>
+ <input name="mount_point" value="source" />
+
+ <input type="submit" />
+</form>
+{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index cc31741..969dfd2 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -48,7 +48,7 @@ from allura.scripts.trac_export import (
from forgeimporters.base import ToolImporter
from forgetracker.tracker_main import ForgeTrackerApp
-from forgetracker.script.import_tracker import import_tracker
+from forgetracker.scripts.import_tracker import import_tracker
class TracTicketImportSchema(fe.Schema):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/forgeimporters/trac/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/wiki.py b/ForgeImporters/forgeimporters/trac/wiki.py
new file mode 100644
index 0000000..2417863
--- /dev/null
+++ b/ForgeImporters/forgeimporters/trac/wiki.py
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+from datetime import (
+ datetime,
+ timedelta,
+ )
+import tempfile
+
+import formencode as fe
+from formencode import validators as fev
+
+from pylons import tmpl_context as c
+from pylons import app_globals as g
+from tg import (
+ config,
+ expose,
+ redirect,
+ validate,
+ )
+from tg.decorators import (
+ with_trailing_slash,
+ without_trailing_slash,
+ )
+
+from allura.controllers import BaseController
+from allura.lib.decorators import require_post
+from allura.model import ApiTicket
+
+from forgeimporters.base import ToolImporter
+
+from forgewiki.scripts.wiki_from_trac.extractors import WikiExporter
+from forgewiki.scripts.wiki_from_trac.loaders import load_data
+from forgewiki.scripts.wiki_from_trac.wiki_from_trac import WikiFromTrac
+from forgewiki.wiki_main import ForgeWikiApp
+
+
+class TracWikiImportSchema(fe.Schema):
+ trac_url = fev.URL(not_empty=True)
+ mount_point = fev.UnicodeString()
+ mount_label = fev.UnicodeString()
+
+
+class TracWikiImportController(BaseController):
+ @with_trailing_slash
+ @expose('jinja:forgeimporters.trac:templates/wiki/index.html')
+ def index(self, **kw):
+ return {}
+
+ @without_trailing_slash
+ @expose()
+ @require_post()
+ @validate(TracWikiImportSchema(), error_handler=index)
+ def create(self, trac_url, mount_point, mount_label, **kw):
+ app = TracWikiImporter.import_tool(c.project,
+ mount_point=mount_point,
+ mount_label=mount_label,
+ trac_url=trac_url,
+ user=c.user)
+ redirect(app.url())
+
+
+class TracWikiImporter(ToolImporter):
+ target_app = ForgeWikiApp
+ source = 'Trac'
+ controller = TracWikiImportController
+ tool_label = 'Trac Wiki Importer'
+ tool_description = 'Import your wiki from Trac'
+
+ def import_tool(self, project=None, mount_point=None, mount_label=None,
+ trac_url=None, user=None):
+ """ Import Trac wiki into a new Allura Wiki tool.
+
+ """
+ mount_point = mount_point or 'wiki'
+ app = project.install_app(
+ 'Wiki',
+ mount_point=mount_point,
+ mount_label=mount_label or 'Wiki',
+ )
+ api_ticket = ApiTicket(user_id=user._id,
+ capabilities={"import": ["Projects", project.shortname]},
+ expires=datetime.utcnow() + timedelta(minutes=60))
+ options = argparse.Namespace()
+ options.api_key = api_ticket.api_key
+ options.secret_key = api_ticket.secret_key
+ options.project = project.shortname
+ options.wiki = mount_point
+ options.base_url = config['base_url']
+ with tempfile.NamedTemporaryFile() as f:
+ WikiExporter(trac_url, options).export(f)
+ f.flush()
+ load_data(f.name, WikiFromTrac.parser(), options)
+ g.post_event('project_updated')
+ return app
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/63fdf19f/ForgeImporters/setup.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py
index 45a08eb..c4776ba 100644
--- a/ForgeImporters/setup.py
+++ b/ForgeImporters/setup.py
@@ -35,8 +35,11 @@ setup(name='ForgeImporters',
# -*- Entry points: -*-
[allura.project_importers]
google-code = forgeimporters.google.project:GoogleCodeProjectImporter
+ trac = forgeimporters.trac.project:TracProjectImporter
[allura.importers]
google-code-tracker = forgeimporters.google.tracker:GoogleCodeTrackerImporter
google-code-repo = forgeimporters.google.code:GoogleRepoImporter
+ trac-tickets = forgeimporters.trac.tickets:TracTicketImporter
+ trac-wiki = forgeimporters.trac.wiki:TracWikiImporter
""",)
[12/13] git commit: [#6464] Changed tool_label on GC code importer
Posted by jo...@apache.org.
[#6464] Changed tool_label on GC code importer
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/cc16a264
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/cc16a264
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/cc16a264
Branch: refs/heads/cj/6464
Commit: cc16a2642111eb410be3335328f972d6990da1f3
Parents: 8098838
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Wed Aug 7 01:14:26 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Aug 7 17:02:28 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/google/code.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/cc16a264/ForgeImporters/forgeimporters/google/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/code.py b/ForgeImporters/forgeimporters/google/code.py
index c88d42a..0eef03b 100644
--- a/ForgeImporters/forgeimporters/google/code.py
+++ b/ForgeImporters/forgeimporters/google/code.py
@@ -97,7 +97,7 @@ class GoogleRepoImporter(ToolImporter):
target_app = TARGET_APPS
source = 'Google Code'
controller = GoogleRepoImportController
- tool_label = 'Google Code Source Importer'
+ tool_label = 'Source Repository'
tool_description = 'Import your SVN, Git, or Hg repo from Google Code'
def import_tool(self, project, user, project_name=None, mount_point=None,
[07/13] git commit: [#6480] Refactor import_tool() params
Posted by jo...@apache.org.
[#6480] Refactor import_tool() params
- project and user are the only guaranteed params for every
importer, and should therefore be the only positional args
- All other args to import_tool, even project_name (which refers
to the name of the project being imported), should be passed
by keyword.
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/85c29b92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/85c29b92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/85c29b92
Branch: refs/heads/cj/6464
Commit: 85c29b926350a1c6cb5f5178fcabc031a12c45ff
Parents: 2379488
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Wed Aug 7 13:32:14 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Aug 7 16:47:24 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/base.py | 10 +++++-----
ForgeImporters/forgeimporters/google/code.py | 6 +++---
ForgeImporters/forgeimporters/google/tests/test_code.py | 4 +++-
ForgeImporters/forgeimporters/google/tracker.py | 3 ++-
.../forgeimporters/tests/google/test_tracker.py | 5 +++--
ForgeImporters/forgeimporters/tests/test_base.py | 5 +++--
ForgeImporters/forgeimporters/trac/tests/test_tickets.py | 5 ++---
ForgeImporters/forgeimporters/trac/tests/test_wiki.py | 5 ++---
ForgeImporters/forgeimporters/trac/tickets.py | 9 ++++-----
ForgeImporters/forgeimporters/trac/wiki.py | 9 ++++-----
10 files changed, 31 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 7ad720c..3aea8c0 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -50,11 +50,10 @@ class ProjectImportForm(schema.Schema):
@task
-def import_tool(importer_name, project_name, mount_point=None, mount_label=None, **kw):
+def import_tool(importer_name, project_name=None, mount_point=None, mount_label=None, **kw):
importer = ToolImporter.by_name(importer_name)
- importer.import_tool(project=c.project, user=c.user,
- mount_point=mount_point,
- mount_label=mount_label, **kw)
+ importer.import_tool(c.project, c.user, project_name=project_name,
+ mount_point=mount_point, mount_label=mount_label, **kw)
class ProjectImporter(BaseController):
@@ -209,7 +208,8 @@ class ToolImporter(object):
importers[ep.name] = importer()
return importers
- def import_tool(self, project, project_name, mount_point=None, mount_label=None):
+ def import_tool(self, project, user, project_name=None,
+ mount_point=None, mount_label=None, **kw):
"""
Override this method to perform the tool import.
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/google/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/code.py b/ForgeImporters/forgeimporters/google/code.py
index 4457b55..c88d42a 100644
--- a/ForgeImporters/forgeimporters/google/code.py
+++ b/ForgeImporters/forgeimporters/google/code.py
@@ -86,7 +86,7 @@ class GoogleRepoImportController(BaseController):
@require_post()
@validate(GoogleRepoImportSchema(), error_handler=index)
def create(self, gc_project_name, mount_point, mount_label, **kw):
- app = GoogleRepoImporter.import_tool(c.project,
+ app = GoogleRepoImporter().import_tool(c.project, c.user,
project_name=gc_project_name,
mount_point=mount_point,
mount_label=mount_label)
@@ -100,8 +100,8 @@ class GoogleRepoImporter(ToolImporter):
tool_label = 'Google Code Source Importer'
tool_description = 'Import your SVN, Git, or Hg repo from Google Code'
- def import_tool(self, project, project_name, mount_point=None, mount_label=None):
- def import_tool(self, project, project_name, mount_point=None, mount_label=None, **kw):
+ def import_tool(self, project, user, project_name=None, mount_point=None,
+ mount_label=None, **kw):
""" Import a Google Code repo into a new SVN, Git, or Hg Allura tool.
"""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/google/tests/test_code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tests/test_code.py b/ForgeImporters/forgeimporters/google/tests/test_code.py
index fe6943b..13ec0c3 100644
--- a/ForgeImporters/forgeimporters/google/tests/test_code.py
+++ b/ForgeImporters/forgeimporters/google/tests/test_code.py
@@ -62,7 +62,8 @@ class TestGoogleRepoImporter(TestCase):
gcpe.return_value.get_repo_type.return_value = 'git'
get_repo_url.return_value = 'http://remote/clone/url/'
p = self._make_project(gc_proj_name='myproject')
- GoogleRepoImporter().import_tool(p, 'project_name')
+ GoogleRepoImporter().import_tool(p, Mock(name='c.user'),
+ project_name='project_name')
get_repo_url.assert_called_once_with('project_name', 'git')
p.install_app.assert_called_once_with('Git',
mount_point='code',
@@ -89,6 +90,7 @@ class TestGoogleRepoImportController(TestController, TestCase):
@patch('forgeimporters.google.code.GoogleRepoImporter')
def test_create(self, gri):
from allura import model as M
+ gri = gri.return_value
gri.import_tool.return_value = Mock()
gri.import_tool.return_value.url.return_value = '/p/{}/mymount'.format(test_project_with_repo)
params = dict(gc_project_name='poop',
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/google/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tracker.py b/ForgeImporters/forgeimporters/google/tracker.py
index 4d37a16..95f53e4 100644
--- a/ForgeImporters/forgeimporters/google/tracker.py
+++ b/ForgeImporters/forgeimporters/google/tracker.py
@@ -42,7 +42,8 @@ class GoogleCodeTrackerImporter(ToolImporter):
type='select',
)
- def import_tool(self, project, project_name, mount_point=None, mount_label=None):
+ def import_tool(self, project, user, project_name=None, mount_point=None,
+ mount_label=None, **kw):
c.app = project.install_app('tracker', mount_point, mount_label)
c.app.globals.open_status_names = ['New', 'Accepted', 'Started']
c.app.globals.closed_status_names = ['Fixed', 'Verified', 'Invalid', 'Duplicate', 'WontFix', 'Done']
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/tests/google/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tracker.py b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
index d54ac90..e49f279 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
@@ -33,14 +33,15 @@ class TestTrackerImporter(TestCase):
importer.process_labels = mock.Mock()
importer.process_comments = mock.Mock()
importer.postprocess_custom_fields = mock.Mock()
- project = mock.Mock()
+ project, user = mock.Mock(), mock.Mock()
app = project.install_app.return_value
extractor = gdata.return_value
issues = extractor.iter_issues.return_value = [mock.Mock(), mock.Mock()]
tickets = TM.Ticket.new.side_effect = [mock.Mock(), mock.Mock()]
comments = extractor.iter_comments.side_effect = [mock.Mock(), mock.Mock()]
- importer.import_tool(project, 'project_name', 'mount_point', 'mount_label')
+ importer.import_tool(project, user, project_name='project_name',
+ mount_point='mount_point', mount_label='mount_label')
project.install_app.assert_called_once_with('tracker', 'mount_point', 'mount_label')
gdata.assert_called_once_with('project_name')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/tests/test_base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/test_base.py b/ForgeImporters/forgeimporters/tests/test_base.py
index 18b83ac..68be24d 100644
--- a/ForgeImporters/forgeimporters/tests/test_base.py
+++ b/ForgeImporters/forgeimporters/tests/test_base.py
@@ -28,10 +28,11 @@ from .. import base
def test_import_tool(c, by_name):
c.project = mock.Mock(name='project')
c.user = mock.Mock(name='user')
- base.import_tool('importer_name', 'project_name', 'mount_point', 'mount_label')
+ base.import_tool('importer_name', project_name='project_name',
+ mount_point='mount_point', mount_label='mount_label')
by_name.assert_called_once_with('importer_name')
by_name.return_value.import_tool.assert_called_once_with(c.project,
- 'project_name', user=c.user, mount_point='mount_point',
+ c.user, project_name='project_name', mount_point='mount_point',
mount_label='mount_label')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 2cce886..5f88eef 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -45,11 +45,10 @@ class TestTracTicketImporter(TestCase):
project = Mock(name='Project', shortname='myproject')
project.install_app.return_value = app
user = Mock(name='User', _id='id')
- res = importer.import_tool(project=project,
+ res = importer.import_tool(project, user,
mount_point='bugs',
mount_label='Bugs',
- trac_url='http://example.com/trac/url',
- user=user)
+ trac_url='http://example.com/trac/url')
self.assertEqual(res, app)
project.install_app.assert_called_once_with(
'Tickets', mount_point='bugs', mount_label='Bugs')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py b/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
index 71cc8b5..738e49b 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_wiki.py
@@ -50,11 +50,10 @@ class TestWikiTicketImporter(TestCase):
project = Mock(name='Project', shortname='myproject')
project.install_app.return_value = app
user = Mock(name='User', _id='id')
- res = importer.import_tool(project=project,
+ res = importer.import_tool(project, user,
mount_point='pages',
mount_label='Pages',
- trac_url='http://example.com/trac/url',
- user=user)
+ trac_url='http://example.com/trac/url')
self.assertEqual(res, app)
project.install_app.assert_called_once_with(
'Wiki', mount_point='pages', mount_label='Pages')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index 78d8d17..f7d50b4 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -69,11 +69,10 @@ class TracTicketImportController(BaseController):
@require_post()
@validate(TracTicketImportSchema(), error_handler=index)
def create(self, trac_url, mount_point, mount_label, **kw):
- app = TracTicketImporter().import_tool(c.project,
+ app = TracTicketImporter().import_tool(c.project, c.user,
mount_point=mount_point,
mount_label=mount_label,
- trac_url=trac_url,
- user=c.user)
+ trac_url=trac_url)
redirect(app.url())
@@ -84,8 +83,8 @@ class TracTicketImporter(ToolImporter):
tool_label = 'Trac Ticket Importer'
tool_description = 'Import your tickets from Trac'
- def import_tool(self, project=None, mount_point=None, mount_label=None,
- trac_url=None, user=None, **kw):
+ def import_tool(self, project, user, project_name=None, mount_point=None,
+ mount_label=None, trac_url=None, **kw):
""" Import Trac tickets into a new Allura Tracker tool.
"""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/85c29b92/ForgeImporters/forgeimporters/trac/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/wiki.py b/ForgeImporters/forgeimporters/trac/wiki.py
index 00f0d48..300b476 100644
--- a/ForgeImporters/forgeimporters/trac/wiki.py
+++ b/ForgeImporters/forgeimporters/trac/wiki.py
@@ -68,11 +68,10 @@ class TracWikiImportController(BaseController):
@require_post()
@validate(TracWikiImportSchema(), error_handler=index)
def create(self, trac_url, mount_point, mount_label, **kw):
- app = TracWikiImporter().import_tool(c.project,
+ app = TracWikiImporter().import_tool(c.project, c.user,
mount_point=mount_point,
mount_label=mount_label,
- trac_url=trac_url,
- user=c.user)
+ trac_url=trac_url)
redirect(app.url())
@@ -83,8 +82,8 @@ class TracWikiImporter(ToolImporter):
tool_label = 'Trac Wiki Importer'
tool_description = 'Import your wiki from Trac'
- def import_tool(self, project=None, mount_point=None, mount_label=None,
- trac_url=None, user=None, **kw):
+ def import_tool(self, project, user, project_name=None, mount_point=None,
+ mount_label=None, trac_url=None, **kw):
""" Import Trac wiki into a new Allura Wiki tool.
"""
[13/13] git commit: [#6464] Google Code Tracker Importer via web
scraping
Posted by jo...@apache.org.
[#6464] Google Code Tracker Importer via web scraping
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/8098838a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/8098838a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/8098838a
Branch: refs/heads/cj/6464
Commit: 8098838a6397d5c58199ba1f0c9903da3d24d283
Parents: 04cd1ed
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Tue Aug 6 23:47:14 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Aug 7 17:02:28 2013 +0000
----------------------------------------------------------------------
Allura/allura/controllers/site_admin.py | 15 +-
.../allura/templates/site_admin_task_view.html | 8 +
.../forgeimporters/google/__init__.py | 150 +++++++++++++--
ForgeImporters/forgeimporters/google/tasks.py | 8 +-
ForgeImporters/forgeimporters/google/tracker.py | 191 ++++---------------
.../tests/google/test_extractor.py | 19 +-
.../forgeimporters/tests/google/test_tasks.py | 8 +-
.../forgeimporters/tests/google/test_tracker.py | 98 +++++-----
8 files changed, 262 insertions(+), 235 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/Allura/allura/controllers/site_admin.py
----------------------------------------------------------------------
diff --git a/Allura/allura/controllers/site_admin.py b/Allura/allura/controllers/site_admin.py
index 4f8f835..50b0e38 100644
--- a/Allura/allura/controllers/site_admin.py
+++ b/Allura/allura/controllers/site_admin.py
@@ -30,6 +30,7 @@ import tg
from pylons import tmpl_context as c, app_globals as g
from pylons import request
from formencode import validators, Invalid
+from webob.exc import HTTPNotFound
from allura.lib import helpers as h
from allura.lib import validators as v
@@ -321,7 +322,19 @@ class TaskManagerController(object):
config_dict['user'] = user
with h.push_config(c, **config_dict):
task = task.post(*args, **kw)
- redirect('view/%s' % task._id)
+ redirect('../view/%s' % task._id)
+
+ @expose()
+ @require_post()
+ def resubmit(self, task_id):
+ try:
+ task = M.monq_model.MonQTask.query.get(_id=bson.ObjectId(task_id))
+ except bson.errors.InvalidId as e:
+ task = None
+ if task is None:
+ raise HTTPNotFound()
+ task.state = 'ready'
+ redirect('../view/%s' % task._id)
@expose('json:')
def task_doc(self, task_name):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/Allura/allura/templates/site_admin_task_view.html
----------------------------------------------------------------------
diff --git a/Allura/allura/templates/site_admin_task_view.html b/Allura/allura/templates/site_admin_task_view.html
index c107382..e363b8d 100644
--- a/Allura/allura/templates/site_admin_task_view.html
+++ b/Allura/allura/templates/site_admin_task_view.html
@@ -66,6 +66,9 @@
#task_details td.second-column {
border: 0;
}
+ #resubmit-task-form {
+ float: right;
+ }
</style>
{% endblock %}
@@ -73,6 +76,11 @@
{% if not task %}
Task not found
{% else %}
+ {% if task.state in ['error', 'complete'] %}
+ <form id="resubmit-task-form" action="../resubmit/{{task._id}}" method="POST">
+ <input type="submit" value="Re-Submit Task" />
+ </form>
+ {% endif %}
<h2>Task Details</h2>
<table id="task_details">
<tr>
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 57e384b..a2a335d 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -20,6 +20,7 @@ import urllib
import urllib2
from urlparse import urlparse, urljoin
from collections import defaultdict
+from contextlib import closing
try:
from cStringIO import StringIO
except ImportError:
@@ -33,12 +34,32 @@ from allura import model as M
log = logging.getLogger(__name__)
+def _as_text(node, chunks=None):
+ """
+ Similar to node.text, but preserves whitespace around tags,
+ and converts <br/>s to \n.
+ """
+ if chunks is None:
+ chunks = []
+ for n in node:
+ if isinstance(n, basestring):
+ chunks.append(n)
+ elif n.name == 'br':
+ chunks.append('\n')
+ else:
+ _as_text(n, chunks)
+ return ''.join(chunks)
+
+
class GoogleCodeProjectExtractor(object):
+ BASE_URL = 'http://code.google.com'
RE_REPO_TYPE = re.compile(r'(svn|hg|git)')
PAGE_MAP = {
- 'project_info': 'http://code.google.com/p/%s/',
- 'source_browse': 'http://code.google.com/p/%s/source/browse/',
+ 'project_info': BASE_URL + '/p/{project_name}/',
+ 'source_browse': BASE_URL + '/p/{project_name}/source/browse/',
+ 'issues_csv': BASE_URL + '/p/{project_name}/issues/csv?can=1&colspec=ID&start={start}',
+ 'issue': BASE_URL + '/p/{project_name}/issues/detail?id={issue_id}',
}
LICENSE_MAP = defaultdict(lambda:'Other/Proprietary License', {
@@ -56,15 +77,16 @@ class GoogleCodeProjectExtractor(object):
DEFAULT_ICON = 'http://www.gstatic.com/codesite/ph/images/defaultlogo.png'
- def __init__(self, allura_project, gc_project_name, page):
- self.project = allura_project
- self.url = self.PAGE_MAP[page] % urllib.quote(gc_project_name)
+ def __init__(self, project_name, page_name, **kw):
+ self.url = self.PAGE_MAP[page_name].format(
+ project_name=urllib.quote(project_name),
+ **kw)
self.page = BeautifulSoup(urllib2.urlopen(self.url))
- def get_short_description(self):
- self.project.short_description = self.page.find(itemprop='description').string.strip()
+ def get_short_description(self, project):
+ project.short_description = self.page.find(itemprop='description').string.strip()
- def get_icon(self):
+ def get_icon(self, project):
icon_url = urljoin(self.url, self.page.find(itemprop='image').attrMap['src'])
if icon_url == self.DEFAULT_ICON:
return
@@ -75,12 +97,12 @@ class GoogleCodeProjectExtractor(object):
icon_name, fp,
fp_ish.info()['content-type'].split(';')[0], # strip off charset=x extra param,
square=True, thumbnail_size=(48,48),
- thumbnail_meta={'project_id': self.project._id, 'category': 'icon'})
+ thumbnail_meta={'project_id': project._id, 'category': 'icon'})
- def get_license(self):
+ def get_license(self, project):
license = self.page.find(text='Code license').findNext().find('a').string.strip()
trove = M.TroveCategory.query.get(fullname=self.LICENSE_MAP[license])
- self.project.trove_license.append(trove._id)
+ project.trove_license.append(trove._id)
def get_repo_type(self):
repo_type = self.page.find(id="crumb_root")
@@ -92,3 +114,109 @@ class GoogleCodeProjectExtractor(object):
return re_match.group(0)
else:
raise Exception("Unknown repo type: {0}".format(repo_type.text))
+
+ @classmethod
+ def _get_issue_ids_page(cls, project_name, start):
+ url = cls.PAGE_MAP['issues_csv'].format(project_name=project_name, start=start)
+ with closing(urllib2.urlopen(url)) as fp:
+ lines = fp.readlines()[1:] # skip CSV header
+ if not lines[-1].startswith('"'):
+ lines.pop() # skip "next page here" info footer
+ issue_ids = [line.strip('",\n') for line in lines]
+ return issue_ids
+
+ @classmethod
+ def iter_issues(cls, project_name):
+ """
+ Iterate over all issues for a project,
+ using paging to keep the responses reasonable.
+ """
+ start = 0
+ limit = 100
+
+ while True:
+ issue_ids = cls._get_issue_ids_page(project_name, start)
+ if len(issue_ids) <= 0:
+ return
+ for issue_id in issue_ids:
+ yield cls(project_name, 'issue', issue_id=issue_id)
+ start += limit
+
+ def get_issue_summary(self):
+ return self.page.find(id='issueheader').findAll('td', limit=2)[1].span.string.strip()
+
+ def get_issue_description(self):
+ return _as_text(self.page.find(id='hc0').pre)
+
+ def get_issue_created_date(self):
+ return self.page.find(id='hc0').find('span', 'date').get('title')
+
+ def get_issue_mod_date(self):
+ last_update = Comment(self.page.findAll('div', 'issuecomment')[-1])
+ return last_update.created_date
+
+ def get_issue_creator(self):
+ a = self.page.find(id='hc0').find('a', 'userlink')
+ return UserLink(a)
+
+ def get_issue_status(self):
+ return self.page.find(id='issuemeta').find('th', text=re.compile('Status:')).findNext().span.string.strip()
+
+ def get_issue_owner(self):
+ return UserLink(self.page.find(id='issuemeta').find('th', text=re.compile('Owner:')).findNext().a)
+
+ def get_issue_labels(self):
+ label_nodes = self.page.find(id='issuemeta').findAll('a', 'label')
+ return [_as_text(l) for l in label_nodes]
+
+ def get_issue_attachments(self):
+ attachments = self.page.find(id='hc0').find('div', 'attachments')
+ if attachments:
+ return map(Attachment, attachments.findAll('tr'))
+ else:
+ return []
+
+ def iter_comments(self):
+ for comment in self.page.findAll('div', 'issuecomment'):
+ yield Comment(comment)
+
+class UserLink(object):
+ def __init__(self, tag):
+ self.name = tag.string.strip()
+ self.link = urljoin(GoogleCodeProjectExtractor.BASE_URL, tag.get('href'))
+
+class Comment(object):
+ def __init__(self, tag):
+ self.author = UserLink(tag.find('span', 'author').find('a', 'userlink'))
+ self.created_date = tag.find('span', 'date').get('title')
+ self.body = _as_text(tag.find('pre'))
+ self._get_updates(tag)
+ self._get_attachments(tag)
+
+ def _get_updates(self, tag):
+ _updates = tag.find('div', 'updates')
+ if _updates:
+ _strings = _updates.findAll(text=True)
+ updates = (s.strip() for s in _strings if s.strip())
+ self.updates = {field: updates.next() for field in updates}
+ else:
+ self.updates = {}
+
+ def _get_attachments(self, tag):
+ attachments = tag.find('div', 'attachments')
+ if attachments:
+ self.attachments = map(Attachment, attachments.findAll('tr'))
+ else:
+ self.attachments = []
+
+class Attachment(object):
+ def __init__(self, tag):
+ self.filename = _as_text(tag).strip().split()[0]
+ self.url = urljoin(GoogleCodeProjectExtractor.BASE_URL, tag.a.get('href'))
+ self.type = None
+
+ @property
+ def file(self):
+ fp_ish = urllib2.urlopen(self.url)
+ fp = StringIO(fp_ish.read())
+ return fp
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/ForgeImporters/forgeimporters/google/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tasks.py b/ForgeImporters/forgeimporters/google/tasks.py
index 968d9a9..69e7556 100644
--- a/ForgeImporters/forgeimporters/google/tasks.py
+++ b/ForgeImporters/forgeimporters/google/tasks.py
@@ -27,9 +27,9 @@ from . import GoogleCodeProjectExtractor
@task
def import_project_info(project_name):
- extractor = GoogleCodeProjectExtractor(c.project, project_name, 'project_info')
- extractor.get_short_description()
- extractor.get_icon()
- extractor.get_license()
+ extractor = GoogleCodeProjectExtractor(project_name, 'project_info')
+ extractor.get_short_description(c.project)
+ extractor.get_icon(c.project)
+ extractor.get_license(c.project)
ThreadLocalORMSession.flush_all()
g.post_event('project_updated')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/ForgeImporters/forgeimporters/google/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tracker.py b/ForgeImporters/forgeimporters/google/tracker.py
index 95f53e4..297f65a 100644
--- a/ForgeImporters/forgeimporters/google/tracker.py
+++ b/ForgeImporters/forgeimporters/google/tracker.py
@@ -19,15 +19,14 @@ from collections import defaultdict
from datetime import datetime
from pylons import tmpl_context as c
-#import gdata
-gdata = None
-from ming.orm import session
+from ming.orm import session, ThreadLocalORMSession
from allura.lib import helpers as h
from forgetracker.tracker_main import ForgeTrackerApp
from forgetracker import model as TM
from ..base import ToolImporter
+from . import GoogleCodeProjectExtractor
class GoogleCodeTrackerImporter(ToolImporter):
@@ -42,23 +41,22 @@ class GoogleCodeTrackerImporter(ToolImporter):
type='select',
)
- def import_tool(self, project, user, project_name=None, mount_point=None,
+ def import_tool(self, project, user, project_name, mount_point=None,
mount_label=None, **kw):
- c.app = project.install_app('tracker', mount_point, mount_label)
- c.app.globals.open_status_names = ['New', 'Accepted', 'Started']
- c.app.globals.closed_status_names = ['Fixed', 'Verified', 'Invalid', 'Duplicate', 'WontFix', 'Done']
+ c.app = project.install_app('tickets', mount_point, mount_label)
+ ThreadLocalORMSession.flush_all()
+ c.app.globals.open_status_names = 'New Accepted Started'
+ c.app.globals.closed_status_names = 'Fixed Verified Invalid Duplicate WontFix Done'
self.custom_fields = {}
- extractor = GDataAPIExtractor(project_name)
- for issue in extractor.iter_issues():
+ for issue in GoogleCodeProjectExtractor.iter_issues(project_name):
ticket = TM.Ticket.new()
self.process_fields(ticket, issue)
self.process_labels(ticket, issue)
- self.process_comments(ticket, extractor.iter_comments(issue))
+ self.process_comments(ticket, issue)
session(ticket).flush(ticket)
session(ticket).expunge(ticket)
self.postprocess_custom_fields()
- session(c.app).flush(c.app)
- session(c.app.globals).flush(c.app.globals)
+ ThreadLocalORMSession.flush_all()
def custom_field(self, name):
if name not in self.custom_fields:
@@ -71,16 +69,25 @@ class GoogleCodeTrackerImporter(ToolImporter):
return self.custom_fields[name]
def process_fields(self, ticket, issue):
- ticket.summary = issue.summary
- ticket.description = issue.description
- ticket.status = issue.status
- ticket.created_date = datetime.strptime(issue.created_date, '')
- ticket.mod_date = datetime.strptime(issue.mod_date, '')
+ ticket.summary = issue.get_issue_summary()
+ ticket.status = issue.get_issue_status()
+ ticket.created_date = datetime.strptime(issue.get_issue_created_date(), '%c')
+ ticket.mod_date = datetime.strptime(issue.get_issue_mod_date(), '%c')
+ ticket.description = (
+ u'*Originally created by:* [{creator.name}]({creator.link})\n'
+ '*Originally owned by:* [{owner.name}]({owner.link})\n'
+ '\n'
+ '{body}').format(
+ creator=issue.get_issue_creator(),
+ owner=issue.get_issue_owner(),
+ body=issue.get_issue_description(),
+ )
+ ticket.add_multiple_attachments(issue.get_issue_attachments())
def process_labels(self, ticket, issue):
labels = set()
custom_fields = defaultdict(set)
- for label in issue.labels:
+ for label in issue.get_issue_labels():
if u'-' in label:
name, value = label.split(u'-', 1)
cf = self.custom_field(name)
@@ -91,23 +98,24 @@ class GoogleCodeTrackerImporter(ToolImporter):
ticket.labels = list(labels)
ticket.custom_fields = {n: u', '.join(sorted(v)) for n,v in custom_fields.iteritems()}
- def process_comments(self, ticket, comments):
- for comment in comments:
- p = ticket.thread.add_post(
+ def process_comments(self, ticket, issue):
+ for comment in issue.iter_comments():
+ p = ticket.discussion_thread.add_post(
text = (
- u'Originally posted by: [{author.name}]({author.link})\n'
+ u'*Originally posted by:* [{author.name}]({author.link})\n'
'\n'
'{body}\n'
'\n'
'{updates}').format(
author=comment.author,
- body=comment.text,
+ body=comment.body,
updates='\n'.join(
- '*%s*: %s' % (k,v)
+ '**%s** %s' % (k,v)
for k,v in comment.updates.items()
),
)
)
+ p.created_date = p.timestamp = datetime.strptime(comment.created_date, '%c')
p.add_multiple_attachments(comment.attachments)
def postprocess_custom_fields(self):
@@ -125,138 +133,3 @@ class GoogleCodeTrackerImporter(ToolImporter):
else:
field['options'] = ''
c.app.globals.custom_fields.append(field)
-
-
-class GDataAPIExtractor(object):
- def __init__(self, project_name):
- self.project_name = project_name
-
- def iter_issues(self, limit=50):
- """
- Iterate over all issues for a project,
- using paging to keep the responses reasonable.
- """
- start = 1
-
- client = gdata.projecthosting.client.ProjectHostingClient()
- while True:
- query = gdata.projecthosting.client.Query(start_index=start, max_results=limit)
- issues = client.get_issues(self.project_name, query=query).entry
- if len(issues) <= 0:
- return
- for issue in issues:
- yield GDataAPIIssue(issue)
- start += limit
-
- def iter_comments(self, issue, limit=50):
- """
- Iterate over all comments for a given issue,
- using paging to keep the responses reasonable.
- """
- start = 1
-
- client = gdata.projecthosting.client.ProjectHostingClient()
- while True:
- query = gdata.projecthosting.client.Query(start_index=start, max_results=limit)
- comments = client.get_comments(self.project_name, query=query).entry
- if len(comments) <= 0:
- return
- for comment in comments:
- yield GDataAPIComment(comment)
- start += limit
-
-
-class GDataAPIUser(object):
- def __init__(self, user):
- self.user = user
-
- @property
- def name(self):
- return h.really_unicode(self.user.name.text)
-
- @property
- def link(self):
- return u'http://code.google.com/u/%s' % self.name
-
-
-class GDataAPIIssue(object):
- def __init__(self, issue):
- self.issue = issue
-
- @property
- def summary(self):
- return h.really_unicode(self.issue.title.text)
-
- @property
- def description(self):
- return h.really_unicode(self.issue.content.text)
-
- @property
- def created_date(self):
- return self.to_date(self.issue.published.text)
-
- @property
- def mod_date(self):
- return self.to_date(self.issue.updated.text)
-
- @property
- def creator(self):
- return h.really_unicode(self.issue.author[0].name.text)
-
- @property
- def status(self):
- if getattr(self.issue, 'status', None) is not None:
- return h.really_unicode(self.issue.status.text)
- return u''
-
- @property
- def owner(self):
- if getattr(self.issue, 'owner', None) is not None:
- return h.really_unicode(self.issue.owner.username.text)
- return u''
-
- @property
- def labels(self):
- return [h.really_unicode(l.text) for l in self.issue.labels]
-
-
-class GDataAPIComment(object):
- def __init__(self, comment):
- self.comment = comment
-
- @property
- def author(self):
- return GDataAPIUser(self.comment.author[0])
-
- @property
- def created_date(self):
- return h.really_unicode(self.comment.published.text)
-
- @property
- def body(self):
- return h.really_unicode(self.comment.content.text)
-
- @property
- def updates(self):
- return {}
-
- @property
- def attachments(self):
- return []
-
-
-class GDataAPIAttachment(object):
- def __init__(self, attachment):
- self.attachment = attachment
-
- @property
- def filename(self):
- pass
-
- @property
- def type(self):
- pass
-
- @property
- def file(self):
- pass
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index 1a3a87c..a2e4183 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -36,18 +36,17 @@ class TestGoogleCodeProjectExtractor(TestCase):
self._p_soup.stop()
def test_init(self):
- extractor = google.GoogleCodeProjectExtractor(self.project, 'my-project', 'project_info')
+ extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
self.urlopen.assert_called_once_with('http://code.google.com/p/my-project/')
- self.assertEqual(extractor.project, self.project)
self.soup.assert_called_once_with(self.urlopen.return_value)
self.assertEqual(extractor.page, self.soup.return_value)
def test_get_short_description(self):
- extractor = google.GoogleCodeProjectExtractor(self.project, 'my-project', 'project_info')
+ extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
extractor.page.find.return_value.string = 'My Super Project'
- extractor.get_short_description()
+ extractor.get_short_description(self.project)
extractor.page.find.assert_called_once_with(itemprop='description')
self.assertEqual(self.project.short_description, 'My Super Project')
@@ -56,11 +55,11 @@ class TestGoogleCodeProjectExtractor(TestCase):
@mock.patch.object(google, 'M')
def test_get_icon(self, M, StringIO):
self.urlopen.return_value.info.return_value = {'content-type': 'image/png'}
- extractor = google.GoogleCodeProjectExtractor(self.project, 'my-project', 'project_info')
+ extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
extractor.page.find.return_value.attrMap = {'src': 'http://example.com/foo/bar/my-logo.png'}
self.urlopen.reset_mock()
- extractor.get_icon()
+ extractor.get_icon(self.project)
extractor.page.find.assert_called_once_with(itemprop='image')
self.urlopen.assert_called_once_with('http://example.com/foo/bar/my-logo.png')
@@ -74,11 +73,11 @@ class TestGoogleCodeProjectExtractor(TestCase):
@mock.patch.object(google, 'M')
def test_get_license(self, M):
self.project.trove_license = []
- extractor = google.GoogleCodeProjectExtractor(self.project, 'my-project', 'project_info')
+ extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
extractor.page.find.return_value.findNext.return_value.find.return_value.string = ' New BSD License '
trove = M.TroveCategory.query.get.return_value
- extractor.get_license()
+ extractor.get_license(self.project)
extractor.page.find.assert_called_once_with(text='Code license')
extractor.page.find.return_value.findNext.assert_called_once_with()
@@ -88,13 +87,13 @@ class TestGoogleCodeProjectExtractor(TestCase):
M.TroveCategory.query.get.reset_mock()
extractor.page.find.return_value.findNext.return_value.find.return_value.string = 'non-existant license'
- extractor.get_license()
+ extractor.get_license(self.project)
M.TroveCategory.query.get.assert_called_once_with(fullname='Other/Proprietary License')
def _make_extractor(self, html):
from BeautifulSoup import BeautifulSoup
with mock.patch.object(google, 'urllib2') as urllib2:
- extractor = google.GoogleCodeProjectExtractor(self.project, 'my-project', 'project_info')
+ extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
extractor.page = BeautifulSoup(html)
extractor.url="http://test/source/browse"
return extractor
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/ForgeImporters/forgeimporters/tests/google/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tasks.py b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
index dc7d936..01bab68 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_tasks.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
@@ -26,8 +26,8 @@ from ...google import tasks
def test_import_project_info(c, session, gpe):
c.project = mock.Mock(name='project')
tasks.import_project_info('my-project')
- gpe.assert_called_once_with(c.project, 'my-project', 'project_info')
- gpe.return_value.get_short_description.assert_called_once_with()
- gpe.return_value.get_icon.assert_called_once_with()
- gpe.return_value.get_license.assert_called_once_with()
+ gpe.assert_called_once_with('my-project', 'project_info')
+ gpe.return_value.get_short_description.assert_called_once_with(c.project)
+ gpe.return_value.get_icon.assert_called_once_with(c.project)
+ gpe.return_value.get_license.assert_called_once_with(c.project)
session.flush_all.assert_called_once_with()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8098838a/ForgeImporters/forgeimporters/tests/google/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tracker.py b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
index e49f279..62493bd 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+from datetime import datetime
from operator import itemgetter
from unittest import TestCase
import mock
@@ -24,10 +25,11 @@ from ...google import tracker
class TestTrackerImporter(TestCase):
@mock.patch.object(tracker, 'c')
+ @mock.patch.object(tracker, 'ThreadLocalORMSession')
@mock.patch.object(tracker, 'session')
@mock.patch.object(tracker, 'TM')
- @mock.patch.object(tracker, 'GDataAPIExtractor')
- def test_import_tool(self, gdata, TM, session, c):
+ @mock.patch.object(tracker, 'GoogleCodeProjectExtractor')
+ def test_import_tool(self, gpe, TM, session, tlos, c):
importer = tracker.GoogleCodeTrackerImporter()
importer.process_fields = mock.Mock()
importer.process_labels = mock.Mock()
@@ -35,16 +37,14 @@ class TestTrackerImporter(TestCase):
importer.postprocess_custom_fields = mock.Mock()
project, user = mock.Mock(), mock.Mock()
app = project.install_app.return_value
- extractor = gdata.return_value
- issues = extractor.iter_issues.return_value = [mock.Mock(), mock.Mock()]
+ issues = gpe.iter_issues.return_value = [mock.Mock(), mock.Mock()]
tickets = TM.Ticket.new.side_effect = [mock.Mock(), mock.Mock()]
- comments = extractor.iter_comments.side_effect = [mock.Mock(), mock.Mock()]
importer.import_tool(project, user, project_name='project_name',
mount_point='mount_point', mount_label='mount_label')
- project.install_app.assert_called_once_with('tracker', 'mount_point', 'mount_label')
- gdata.assert_called_once_with('project_name')
+ project.install_app.assert_called_once_with('tickets', 'mount_point', 'mount_label')
+ gpe.iter_issues.assert_called_once_with('project_name')
self.assertEqual(importer.process_fields.call_args_list, [
mock.call(tickets[0], issues[0]),
mock.call(tickets[1], issues[1]),
@@ -54,26 +54,16 @@ class TestTrackerImporter(TestCase):
mock.call(tickets[1], issues[1]),
])
self.assertEqual(importer.process_comments.call_args_list, [
- mock.call(tickets[0], comments[0]),
- mock.call(tickets[1], comments[1]),
+ mock.call(tickets[0], issues[0]),
+ mock.call(tickets[1], issues[1]),
])
- self.assertEqual(extractor.iter_comments.call_args_list, [
- mock.call(issues[0]),
- mock.call(issues[1]),
- ])
- self.assertEqual(session.call_args_list, [
- mock.call(tickets[0]),
- mock.call(tickets[0]),
- mock.call(tickets[1]),
- mock.call(tickets[1]),
- mock.call(app),
- mock.call(app.globals),
+ self.assertEqual(tlos.flush_all.call_args_list, [
+ mock.call(),
+ mock.call(),
])
self.assertEqual(session.return_value.flush.call_args_list, [
mock.call(tickets[0]),
mock.call(tickets[1]),
- mock.call(app),
- mock.call(app.globals),
])
self.assertEqual(session.return_value.expunge.call_args_list, [
mock.call(tickets[0]),
@@ -119,30 +109,37 @@ class TestTrackerImporter(TestCase):
def test_process_fields(self):
ticket = mock.Mock()
+ def _user(l):
+ u = mock.Mock()
+ u.name = '%sname' % l
+ u.link = '%slink' % l
+ return u
issue = mock.Mock(
- summary='summary',
- description='description',
- status='status',
- created_date='created_date',
- mod_date='mod_date',
+ get_issue_summary=lambda:'summary',
+ get_issue_description=lambda:'description',
+ get_issue_status=lambda:'status',
+ get_issue_created_date=lambda:'created_date',
+ get_issue_mod_date=lambda:'mod_date',
+ get_issue_creator=lambda:_user('c'),
+ get_issue_owner=lambda:_user('o'),
)
importer = tracker.GoogleCodeTrackerImporter()
with mock.patch.object(tracker, 'datetime') as dt:
dt.strptime.side_effect = lambda s,f: s
importer.process_fields(ticket, issue)
self.assertEqual(ticket.summary, 'summary')
- self.assertEqual(ticket.description, 'description')
+ self.assertEqual(ticket.description, '*Originally created by:* [cname](clink)\n*Originally owned by:* [oname](olink)\n\ndescription')
self.assertEqual(ticket.status, 'status')
self.assertEqual(ticket.created_date, 'created_date')
self.assertEqual(ticket.mod_date, 'mod_date')
self.assertEqual(dt.strptime.call_args_list, [
- mock.call('created_date', ''),
- mock.call('mod_date', ''),
+ mock.call('created_date', '%c'),
+ mock.call('mod_date', '%c'),
])
def test_process_labels(self):
ticket = mock.Mock(custom_fields={}, labels=[])
- issue = mock.Mock(labels=['Foo-Bar', 'Baz', 'Foo-Qux'])
+ issue = mock.Mock(get_issue_labels=lambda:['Foo-Bar', 'Baz', 'Foo-Qux'])
importer = tracker.GoogleCodeTrackerImporter()
importer.custom_field = mock.Mock(side_effect=lambda n: {'name': '_%s' % n.lower(), 'options': set()})
importer.process_labels(ticket, issue)
@@ -156,40 +153,49 @@ class TestTrackerImporter(TestCase):
a.link = 'author%s_link' % n
return a
ticket = mock.Mock()
- comments = [
+ issue = mock.Mock()
+ comments = issue.iter_comments.return_value = [
mock.Mock(
author=_author(1),
- text='text1',
+ body='text1',
attachments='attachments1',
+ created_date='Mon Jul 15 00:00:00 2013',
),
mock.Mock(
author=_author(2),
- text='text2',
+ body='text2',
attachments='attachments2',
+ created_date='Mon Jul 16 00:00:00 2013',
),
]
- comments[0].updates.items.return_value = [('Foo', 'Bar'), ('Baz', 'Qux')]
+ comments[0].updates.items.return_value = [('Foo:', 'Bar'), ('Baz:', 'Qux')]
comments[1].updates.items.return_value = []
+ posts = ticket.discussion_thread.add_post.side_effect = [
+ mock.Mock(),
+ mock.Mock(),
+ ]
importer = tracker.GoogleCodeTrackerImporter()
- importer.process_comments(ticket, comments)
- self.assertEqual(ticket.thread.add_post.call_args_list[0], mock.call(
- text='Originally posted by: [author1](author1_link)\n'
+ importer.process_comments(ticket, issue)
+ self.assertEqual(ticket.discussion_thread.add_post.call_args_list[0], mock.call(
+ text='*Originally posted by:* [author1](author1_link)\n'
'\n'
'text1\n'
'\n'
- '*Foo*: Bar\n'
- '*Baz*: Qux'
+ '**Foo:** Bar\n'
+ '**Baz:** Qux'
))
- self.assertEqual(ticket.thread.add_post.call_args_list[1], mock.call(
- text='Originally posted by: [author2](author2_link)\n'
+ self.assertEqual(posts[0].created_date, datetime(2013, 7, 15))
+ self.assertEqual(posts[0].timestamp, datetime(2013, 7, 15))
+ posts[0].add_multiple_attachments.assert_called_once_with('attachments1')
+ self.assertEqual(ticket.discussion_thread.add_post.call_args_list[1], mock.call(
+ text='*Originally posted by:* [author2](author2_link)\n'
'\n'
'text2\n'
'\n'
))
- self.assertEqual(ticket.thread.add_post.return_value.add_multiple_attachments.call_args_list, [
- mock.call('attachments1'),
- mock.call('attachments2'),
- ])
+ self.assertEqual(posts[1].created_date, datetime(2013, 7, 16))
+ self.assertEqual(posts[1].timestamp, datetime(2013, 7, 16))
+ posts[1].add_multiple_attachments.assert_called_once_with('attachments2')
@mock.patch.object(tracker, 'c')
def test_postprocess_custom_fields(self, c):