You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2013/11/13 20:15:48 UTC
[2/3] git commit: [#6845] Preserve inter-project links when importing
from GC
[#6845] Preserve inter-project links when importing from GC
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/a069ca45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/a069ca45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/a069ca45
Branch: refs/heads/master
Commit: a069ca45307c1e4aafee00d0620130221017ddce
Parents: 7ed59a1
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Tue Nov 12 18:13:26 2013 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Wed Nov 13 19:15:29 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/google/__init__.py | 36 +++++++++++++++-----
.../tests/data/google/test-issue.html | 2 +-
.../tests/google/functional/test_tracker.py | 4 +--
.../tests/google/test_extractor.py | 6 ++--
4 files changed, 34 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/a069ca45/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 261b9f9..7a75eaf 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -52,21 +52,41 @@ def _as_text(node, chunks=None):
_as_text(n, chunks)
return ''.join(chunks)
-def _as_markdown(tag):
+def _as_markdown(tag, project_name):
fragments = []
for fragment in tag:
if getattr(fragment, 'name', None) == 'a':
href = urlparse(fragment['href'])
qs = parse_qs(href.query)
- if not href.netloc and 'id' in qs:
+ gc_link = not href.netloc or href.netloc == 'code.google.com'
+ target_project = href.path.split('/')[2]
+ internal_link = target_project == project_name
+ if gc_link and internal_link and 'id' in qs:
+ # rewrite issue 123 project-internal issue links
fragment = '[%s](#%s)' % (fragment.text, qs['id'][0])
- elif not href.netloc and 'r' in qs:
+ elif gc_link and internal_link and 'r' in qs:
+ # rewrite r123 project-internal revision links
fragment = '[r%s]' % qs['r'][0]
+ elif gc_link:
+ # preserve GC-internal links (probably issue PROJECT:123 inter-project issue links)
+ fragment = '[%s](%s)' % (
+ h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True),
+ urljoin('https://code.google.com/p/%s/issues/' % project_name, fragment['href']),
+ )
else:
+ # un-link all others
fragment = h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True)
elif getattr(fragment, 'name', None) == 'i':
+ # preserve styling of "(No comment was entered for this change.)" messages
fragment = '*%s*' % h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True)
+ elif getattr(fragment, 'name', None) == 'b':
+ # preserve styling of issue template
+ fragment = '**%s**' % h.plain2markdown(fragment.text, preserve_multiple_spaces=True, has_html_entities=True)
+ elif getattr(fragment, 'name', None) == 'br':
+ # preserve forced line-breaks
+ fragment = '\n'
else:
+ # convert all others to plain MD
fragment = h.plain2markdown(str(fragment), preserve_multiple_spaces=True, has_html_entities=True)
fragments.append(fragment)
return ''.join(fragments).strip()
@@ -203,7 +223,7 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
return bs.text
def get_issue_description(self):
- return _as_markdown(self.page.find(id='hc0').pre)
+ return _as_markdown(self.page.find(id='hc0').pre, self.project_name)
def get_issue_created_date(self):
return self.page.find(id='hc0').find('span', 'date').get('title')
@@ -211,7 +231,7 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
def get_issue_mod_date(self):
comments = self.page.findAll('div', 'issuecomment')
if comments:
- last_update = Comment(comments[-1])
+ last_update = Comment(comments[-1], self.project_name)
return last_update.created_date
else:
return self.get_issue_created_date()
@@ -250,7 +270,7 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
def iter_comments(self):
for comment in self.page.findAll('div', 'issuecomment'):
- yield Comment(comment)
+ yield Comment(comment, self.project_name)
class UserLink(object):
def __init__(self, tag):
@@ -283,10 +303,10 @@ def _get_attachments(tag):
return []
class Comment(object):
- def __init__(self, tag):
+ def __init__(self, tag, project_name):
self.author = UserLink(tag.find('span', 'author').find(True, 'userlink'))
self.created_date = tag.find('span', 'date').get('title')
- self.body = _as_markdown(tag.find('pre'))
+ self.body = _as_markdown(tag.find('pre'), project_name)
self._get_updates(tag)
self.attachments = _get_attachments(tag)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/a069ca45/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/data/google/test-issue.html b/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
index 374c0e8..c6bce0a 100644
--- a/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
+++ b/ForgeImporters/forgeimporters/tests/data/google/test-issue.html
@@ -452,7 +452,7 @@ Last comment
<a name="c4" href="/p/allura-google-importer/issues/detail?id=6#c4">#4</a>
<a class="userlink" href="/u/101557263855536553789/">john...@gmail.com</a></span>
<pre>
-Oh, I forgot one
+Oh, I forgot one (with an inter-project reference to <a href="/p/other-project/issues/detail?id=1">issue other-project:1</a>)
</pre>
<div class="updates">
<div class="round4"></div>
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/a069ca45/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py b/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
index 2e52be6..c22f7d0 100644
--- a/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
@@ -41,7 +41,7 @@ class TestGCTrackerImporter(TestCase):
def _make_extractor(self, html):
with mock.patch.object(base.h, 'urlopen') as urlopen:
urlopen.return_value = ''
- extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
+ extractor = google.GoogleCodeProjectExtractor('allura-google-importer', 'project_info')
extractor.page = BeautifulSoup(html)
extractor.url = "http://test/issue/?id=1"
return extractor
@@ -241,7 +241,7 @@ class TestGCTrackerImporter(TestCase):
'text': (
'*Originally posted by:* [john...@gmail.com](http://code.google.com/u/101557263855536553789/)\n'
'\n'
- 'Oh, I forgot one\n'
+ 'Oh, I forgot one \\(with an inter\\-project reference to [issue other\\-project:1](https://code.google.com/p/other-project/issues/detail?id=1)\\)\n'
'\n'
'**Labels:** OpSys-OSX'
),
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/a069ca45/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index 032e122..78bde81 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -124,7 +124,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
def _make_extractor(self, html):
from BeautifulSoup import BeautifulSoup
with mock.patch.object(base.ProjectExtractor, 'urlopen'):
- extractor = google.GoogleCodeProjectExtractor('my-project')
+ extractor = google.GoogleCodeProjectExtractor('allura-google-importer')
extractor.page = BeautifulSoup(html)
extractor.get_page = lambda pagename: extractor.page
extractor.url="http://test/source/browse"
@@ -278,7 +278,7 @@ class TestGoogleCodeProjectExtractor(TestCase):
'author.name': 'john...@gmail.com',
'author.url': 'http://code.google.com/u/101557263855536553789/',
'created_date': 'Thu Aug 8 15:36:57 2013',
- 'body': 'Oh, I forgot one',
+ 'body': 'Oh, I forgot one \\(with an inter-project reference to [issue other-project:1](https://code.google.com/p/other-project/issues/detail?id=1)\\)',
'updates': {'Labels:': 'OpSys-OSX'},
'attachments': [],
},
@@ -390,7 +390,7 @@ class TestComment(TestCase):
def test_init(self):
from BeautifulSoup import BeautifulSoup
html = BeautifulSoup(self.html)
- comment = google.Comment(html.find('div', 'issuecomment'))
+ comment = google.Comment(html.find('div', 'issuecomment'), 'pychess')
self.assertEqual(comment.updates, {
u'Summary:': u'Make PyChess keyboard accessible',
u'Status:': u'Accepted',