You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/08/26 21:11:12 UTC
[1/7] git commit: [#6531] Fixed anonymous comments in trac ticket
export
Updated Branches:
refs/heads/master 1843655af -> 8c6605d3f
[#6531] Fixed anonymous comments in trac ticket export
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/0688afc8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/0688afc8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/0688afc8
Branch: refs/heads/master
Commit: 0688afc859914835de42bdd3118c5b5e828e9c56
Parents: ff5af16
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Mon Aug 26 17:05:30 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Aug 26 17:19:57 2013 +0000
----------------------------------------------------------------------
Allura/allura/scripts/trac_export.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0688afc8/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
index 4d908bc..43462df 100644
--- a/Allura/allura/scripts/trac_export.py
+++ b/Allura/allura/scripts/trac_export.py
@@ -155,7 +155,7 @@ class TracExport(object):
res = []
for comment in d['entries']:
c = {}
- c['submitter'] = comment.author
+ c['submitter'] = getattr(comment, 'author', None)
c['date'] = comment.updated_parsed
c['comment'] = html2text(comment.summary)
c['class'] = 'COMMENT'
[4/7] git commit: [#6531] ticket:413 Project metadata importer for
github
Posted by jo...@apache.org.
[#6531] ticket:413 Project metadata importer for github
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/69db59c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/69db59c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/69db59c2
Branch: refs/heads/master
Commit: 69db59c23bf18cd9f53301bb3c048cdadf2b9e38
Parents: 1843655
Author: Yuriy Arhipov <yu...@yandex.ru>
Authored: Tue Aug 20 13:18:20 2013 +0400
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Aug 26 17:19:57 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/github/__init__.py | 43 +++++++++++++
ForgeImporters/forgeimporters/github/project.py | 66 ++++++++++++++++++++
ForgeImporters/forgeimporters/github/tasks.py | 31 +++++++++
.../github/templates/project.html | 55 ++++++++++++++++
.../forgeimporters/tests/github/__init__.py | 17 +++++
.../tests/github/functional/__init__.py | 17 +++++
.../tests/github/functional/test_github.py | 28 +++++++++
.../tests/github/test_extractor.py | 48 ++++++++++++++
.../forgeimporters/tests/github/test_tasks.py | 31 +++++++++
ForgeImporters/setup.py | 1 +
10 files changed, 337 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
new file mode 100644
index 0000000..472429e
--- /dev/null
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import re
+import urllib
+import urllib2
+import json
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+import logging
+
+log = logging.getLogger(__name__)
+
+class GitHubProjectExtractor(object):
+ RE_REPO_TYPE = re.compile(r'(svn|hg|git)')
+ PAGE_MAP = {
+ 'project_info': 'https://api.github.com/repos/%s',
+ }
+
+
+ def __init__(self, allura_project, gh_project_name, page):
+ self.project = allura_project
+ self.url = self.PAGE_MAP[page] % urllib.quote(gh_project_name)
+ self.page = json.loads(urllib2.urlopen(self.url).read().decode('utf8'))
+
+ def get_summmary(self):
+ self.project.summary = self.page['description']
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/github/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/project.py b/ForgeImporters/forgeimporters/github/project.py
new file mode 100644
index 0000000..4af2bdf
--- /dev/null
+++ b/ForgeImporters/forgeimporters/github/project.py
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from formencode import validators as fev
+
+from tg import expose, validate
+from tg.decorators import with_trailing_slash
+
+from allura.lib.decorators import require_post
+
+from .. import base
+from . import tasks
+
+
+log = logging.getLogger(__name__)
+
+class GitHubProjectForm(base.ProjectImportForm):
+ project_name = fev.Regex(r'^[a-z0-9][a-z0-9-]{,61}$',
+ not_empty=True,
+ messages={
+ 'invalid': 'Please use only letters, numbers, and dashes.',
+ })
+
+class GitHubProjectImporter(base.ProjectImporter):
+
+ source = 'GitHub'
+ index_template = 'jinja:forgeimporters.github:templates/project.html'
+ process_validator = GitHubProjectForm(source)
+
+ def after_project_create(self, project, **kw):
+ project.set_tool_data('github', project_name=project.name)
+ tasks.import_project_info.post(project.name)
+
+ @with_trailing_slash
+ @expose(index_template)
+ def index(self, **kw):
+ return super(self.__class__, self).index(**kw)
+
+ @require_post()
+ @expose()
+ @validate(process_validator)
+ def process(self, **kw):
+ kw['project_name'] = '%s/%s' % (kw['user_name'], kw['project_name'])
+ kw['tools'] = ''
+ return super(self.__class__, self).process(**kw)
+
+ @expose('json:')
+ @validate(process_validator)
+ def check_names(self, **kw):
+ return super(self.__class__, self).check_names(**kw)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/github/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tasks.py b/ForgeImporters/forgeimporters/github/tasks.py
new file mode 100644
index 0000000..4fd928c
--- /dev/null
+++ b/ForgeImporters/forgeimporters/github/tasks.py
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pylons import tmpl_context as c
+
+from ming.orm import ThreadLocalORMSession
+
+from allura.lib.decorators import task
+
+from . import GitHubProjectExtractor
+
+
+@task
+def import_project_info(project_name):
+ extractor = GitHubProjectExtractor(c.project, project_name, 'project_info')
+ extractor.get_summmary()
+ ThreadLocalORMSession.flush_all()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/github/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/templates/project.html b/ForgeImporters/forgeimporters/github/templates/project.html
new file mode 100644
index 0000000..de42737
--- /dev/null
+++ b/ForgeImporters/forgeimporters/github/templates/project.html
@@ -0,0 +1,55 @@
+{#-
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
+{% extends 'forgeimporters:templates/project_base.html' %}
+
+{% block project_fields %}
+ <div class="grid-6">
+ <label>GitHub User Name</label>
+ </div>
+ <div class="grid-10">
+ <input id="user_name" name="user_name" value="{{c.form_values['user_name']}}" autofocus/>
+ <div id="user_name_error" class="error{% if not c.form_errors['user_name'] %} hidden{% endif %}">
+ {{c.form_errors['user_name']}}
+ </div>
+ </div>
+
+
+ <div class="grid-6" style="clear:left">
+ <label>GitHub Project Name</label>
+ </div>
+ <div class="grid-10">
+ <input id="project_name" name="project_name" value="{{c.form_values['project_name']}}" autofocus/>
+ <div id="project_name_error" class="error{% if not c.form_errors['project_name'] %} hidden{% endif %}">
+ {{c.form_errors['project_name']}}
+ </div>
+ </div>
+
+ <div class="grid-6" style="clear:left">
+ <label>URL Name</label>
+ </div>
+ <div class="grid-10">
+ <input id="project_shortname" name="project_shortname" value="{{c.form_values['project_shortname']}}"/>
+ <div id="project_shortname_error" class="error{% if not c.form_errors['project_shortname'] %} hidden{% endif %}">
+ {{c.form_errors['project_shortname']}}
+ </div>
+ <div id="project-url">
+ http://{{request.environ['HTTP_HOST']}}{{importer.neighborhood.url()}}<span id="url-fragment">{{c.form_values['project_shortname']}}</span>
+ </div>
+ </div>
+{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/tests/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/__init__.py b/ForgeImporters/forgeimporters/tests/github/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/github/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/tests/github/functional/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/functional/__init__.py b/ForgeImporters/forgeimporters/tests/github/functional/__init__.py
new file mode 100644
index 0000000..77505f1
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/github/functional/__init__.py
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/functional/test_github.py b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
new file mode 100644
index 0000000..6667e7b
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+from allura.tests import TestController
+
+class TestGitHubImportController(TestController, TestCase):
+
+ def test_index(self):
+ r = self.app.get('/p/import_project/github/')
+ assert 'GitHub Project Importer' in r
+ assert '<input id="user_name" name="user_name" value="" autofocus/>' in r
+ assert '<input id="project_name" name="project_name" value="" autofocus/>' in r
+ assert '<input id="project_shortname" name="project_shortname" value=""/>' in r
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
new file mode 100644
index 0000000..a21f775
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+
+import mock
+
+from ... import github
+
+
+class TestGitHubProjectExtractor(TestCase):
+ def setUp(self):
+ self._p_urlopen = mock.patch.object(github.urllib2, 'urlopen')
+ self._p_json = mock.patch.object(github.json, 'loads')
+ self.urlopen = self._p_urlopen.start()
+ self.json = self._p_json.start()
+ self.project = mock.Mock(name='project')
+ self.project.get_tool_data.return_value = 'testproject'
+
+ def tearDown(self):
+ self._p_urlopen.stop()
+ self._p_json.stop()
+
+
+ def test_init(self):
+ extractor = github.GitHubProjectExtractor(self.project, 'testproject', 'project_info')
+ self.urlopen.assert_called_once_with('https://api.github.com/repos/testproject')
+ self.assertEqual(extractor.project, self.project)
+
+ def test_get_summary(self):
+ extractor = github.GitHubProjectExtractor(self.project, 'testproject', 'project_info')
+ extractor.page = {'description': 'test summary'}
+ extractor.get_summmary()
+ self.assertEqual(self.project.summary, 'test summary')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/forgeimporters/tests/github/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_tasks.py b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
new file mode 100644
index 0000000..b478bc9
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from ...github import tasks
+
+
+@mock.patch.object(tasks, 'GitHubProjectExtractor')
+@mock.patch.object(tasks, 'ThreadLocalORMSession')
+@mock.patch.object(tasks, 'c')
+def test_import_project_info(c, session, ghpe):
+ c.project = mock.Mock(name='project')
+ tasks.import_project_info('my-project')
+ ghpe.assert_called_once_with(c.project, 'my-project', 'project_info')
+ ghpe.return_value.get_summmary.assert_called_once_with()
+ session.flush_all.assert_called_once_with()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/69db59c2/ForgeImporters/setup.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py
index b19fd17..cec76b4 100644
--- a/ForgeImporters/setup.py
+++ b/ForgeImporters/setup.py
@@ -36,6 +36,7 @@ setup(name='ForgeImporters',
[allura.project_importers]
google-code = forgeimporters.google.project:GoogleCodeProjectImporter
trac = forgeimporters.trac.project:TracProjectImporter
+ github = forgeimporters.github.project:GitHubProjectImporter
[allura.importers]
google-code-tracker = forgeimporters.google.tracker:GoogleCodeTrackerImporter
[6/7] git commit: [#6531] Refactored more stuff up into
ProjectExtractor
Posted by jo...@apache.org.
[#6531] Refactored more stuff up into ProjectExtractor
Signed-off-by: Tim Van Steenburgh <tv...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/2d5cf6c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/2d5cf6c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/2d5cf6c5
Branch: refs/heads/master
Commit: 2d5cf6c51f0aabd5748856cc4b1b1b9c01bb67c1
Parents: 0008c42
Author: Tim Van Steenburgh <tv...@gmail.com>
Authored: Fri Aug 23 16:01:29 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Aug 26 17:19:57 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/base.py | 59 ++++++++++++++++++++
.../forgeimporters/github/__init__.py | 30 ++++------
ForgeImporters/forgeimporters/github/tasks.py | 8 ++-
.../github/templates/project.html | 3 +-
.../forgeimporters/google/__init__.py | 39 -------------
.../tests/github/functional/test_github.py | 2 +-
.../tests/github/test_extractor.py | 32 ++++-------
.../forgeimporters/tests/github/test_tasks.py | 5 +-
.../tests/google/test_extractor.py | 3 +-
9 files changed, 92 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 49397a5..3cf6774 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -16,10 +16,12 @@
# under the License.
import logging
+import urllib
import urllib2
from pkg_resources import iter_entry_points
+from BeautifulSoup import BeautifulSoup
from tg import expose, validate, flash, redirect, config
from tg.decorators import with_trailing_slash
from pylons import tmpl_context as c
@@ -67,12 +69,69 @@ class ProjectExtractor(object):
a custom User-Agent and automatically retries timed-out requests.
"""
+
+ PAGE_MAP = {}
+
+ def __init__(self, project_name, page_name=None, **kw):
+ self.project_name = project_name
+ self._page_cache = {}
+ self.url = None
+ self.page = None
+ if page_name:
+ self.get_page(page_name, **kw)
+
@staticmethod
def urlopen(url, retries=3, codes=(408,), **kw):
req = urllib2.Request(url, **kw)
req.add_header('User-Agent', 'Allura Data Importer (http://sf.net/p/allura)')
return h.urlopen(req, retries=retries, codes=codes)
+ def get_page(self, page_name_or_url, **kw):
+ """Return a Beautiful soup object for the given page name or url.
+
+ If a page name is provided, the associated url is looked up in
+ :attr:`PAGE_MAP`.
+
+ Results are cached so that subsequent calls for the same page name or
+ url will return the cached result rather than making another HTTP
+ request.
+
+ """
+ if page_name_or_url in self.PAGE_MAP:
+ self.url = self.get_page_url(page_name_or_url, **kw)
+ else:
+ self.url = page_name_or_url
+ if self.url in self._page_cache:
+ self.page = self._page_cache[self.url]
+ else:
+ self.page = self._page_cache[self.url] = \
+ self.parse_page(self.urlopen(self.url))
+ return self.page
+
+ def get_page_url(self, page_name, **kw):
+ """Return the url associated with ``page_name``.
+
+ Raises KeyError if ``page_name`` is not in :attr:`PAGE_MAP`.
+
+ """
+ return self.PAGE_MAP[page_name].format(
+ project_name = urllib.quote(self.project_name), **kw)
+
+ def parse_page(self, page):
+ """Transforms the result of a `urlopen` call before returning it from
+ :meth:`get_page`.
+
+ The default implementation create a :class:`BeautifulSoup` object from
+ the html.
+
+ Subclasses can override to change the behavior or handle other types
+ of content (like JSON).
+
+ :param page: A file-like object return from :meth:`urlopen`
+
+ """
+ return BeautifulSoup(page)
+
class ProjectImporter(BaseController):
"""
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 1ce4398..f08f478 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -15,32 +15,24 @@
# specific language governing permissions and limitations
# under the License.
-import re
-import urllib
-import urllib2
-import json
-try:
- from cStringIO import StringIO
-except ImportError:
- from StringIO import StringIO
import logging
+import json
+
+from forgeimporters import base
log = logging.getLogger(__name__)
-class GitHubProjectExtractor(object):
- RE_REPO_TYPE = re.compile(r'(svn|hg|git)')
+
+class GitHubProjectExtractor(base.ProjectExtractor):
PAGE_MAP = {
- 'project_info': 'https://api.github.com/repos/%s',
+ 'project_info': 'https://api.github.com/repos/{project_name}',
}
+ def parse_page(self, page):
+ return json.loads(page.read().decode('utf8'))
- def __init__(self, allura_project, gh_project_name, page):
- self.project = allura_project
- self.url = self.PAGE_MAP[page] % urllib.quote(gh_project_name)
- self.page = json.loads(urllib2.urlopen(self.url).read().decode('utf8'))
-
- def get_summmary(self):
- self.project.summary = self.page['description']
+ def get_summary(self):
+ return self.get_page('project_info').get('description')
def get_homepage(self):
- self.project.external_homepage = self.page['homepage']
+ return self.get_page('project_info').get('homepage')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/github/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tasks.py b/ForgeImporters/forgeimporters/github/tasks.py
index be2325d..588c5a6 100644
--- a/ForgeImporters/forgeimporters/github/tasks.py
+++ b/ForgeImporters/forgeimporters/github/tasks.py
@@ -16,6 +16,7 @@
# under the License.
from pylons import tmpl_context as c
+from pylons import app_globals as g
from ming.orm import ThreadLocalORMSession
@@ -26,7 +27,8 @@ from . import GitHubProjectExtractor
@task
def import_project_info(project_name):
- extractor = GitHubProjectExtractor(c.project, project_name, 'project_info')
- extractor.get_summmary()
- extractor.get_homepage()
+ extractor = GitHubProjectExtractor(project_name)
+ c.project.summary = extractor.get_summary()
+ c.project.external_homepage = extractor.get_homepage()
ThreadLocalORMSession.flush_all()
+ g.post_event('project_updated')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/github/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/templates/project.html b/ForgeImporters/forgeimporters/github/templates/project.html
index de42737..d0f59f6 100644
--- a/ForgeImporters/forgeimporters/github/templates/project.html
+++ b/ForgeImporters/forgeimporters/github/templates/project.html
@@ -29,12 +29,11 @@
</div>
</div>
-
<div class="grid-6" style="clear:left">
<label>GitHub Project Name</label>
</div>
<div class="grid-10">
- <input id="project_name" name="project_name" value="{{c.form_values['project_name']}}" autofocus/>
+ <input id="project_name" name="project_name" value="{{c.form_values['project_name']}}" />
<div id="project_name_error" class="error{% if not c.form_errors['project_name'] %} hidden{% endif %}">
{{c.form_errors['project_name']}}
</div>
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 902683a..29e5011 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -78,45 +78,6 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
DEFAULT_ICON = 'http://www.gstatic.com/codesite/ph/images/defaultlogo.png'
- def __init__(self, project_name, page_name=None, **kw):
- self.project_name = project_name
- self._page_cache = {}
- self.url = None
- self.page = None
- if page_name:
- self.get_page(page_name, **kw)
-
- def get_page(self, page_name_or_url, **kw):
- """Return a Beautiful soup object for the given page name or url.
-
- If a page name is provided, the associated url is looked up in
- :attr:`PAGE_MAP`.
-
- Results are cached so that subsequent calls for the same page name or
- url will return the cached result rather than making another HTTP
- request.
-
- """
- if page_name_or_url in self.PAGE_MAP:
- self.url = self.get_page_url(page_name_or_url, **kw)
- else:
- self.url = page_name_or_url
- if self.url in self._page_cache:
- self.page = self._page_cache[self.url]
- else:
- self.page = self._page_cache[self.url] = \
- BeautifulSoup(self.urlopen(self.url))
- return self.page
-
- def get_page_url(self, page_name, **kw):
- """Return the url associated with ``page_name``.
-
- Raises KeyError if ``page_name`` is not in :attr:`PAGE_MAP`.
-
- """
- return self.PAGE_MAP[page_name].format(
- project_name = urllib.quote(self.project_name), **kw)
-
def get_short_description(self, project):
page = self.get_page('project_info')
project.short_description = page.find(itemprop='description').string.strip()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/functional/test_github.py b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
index 6667e7b..3c05a2d 100644
--- a/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
+++ b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
@@ -24,5 +24,5 @@ class TestGitHubImportController(TestController, TestCase):
r = self.app.get('/p/import_project/github/')
assert 'GitHub Project Importer' in r
assert '<input id="user_name" name="user_name" value="" autofocus/>' in r
- assert '<input id="project_name" name="project_name" value="" autofocus/>' in r
+ assert '<input id="project_name" name="project_name" value="" />' in r
assert '<input id="project_shortname" name="project_shortname" value=""/>' in r
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index a21f775..cffbf81 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -17,32 +17,20 @@
from unittest import TestCase
-import mock
-
from ... import github
class TestGitHubProjectExtractor(TestCase):
def setUp(self):
- self._p_urlopen = mock.patch.object(github.urllib2, 'urlopen')
- self._p_json = mock.patch.object(github.json, 'loads')
- self.urlopen = self._p_urlopen.start()
- self.json = self._p_json.start()
- self.project = mock.Mock(name='project')
- self.project.get_tool_data.return_value = 'testproject'
-
- def tearDown(self):
- self._p_urlopen.stop()
- self._p_json.stop()
-
-
- def test_init(self):
- extractor = github.GitHubProjectExtractor(self.project, 'testproject', 'project_info')
- self.urlopen.assert_called_once_with('https://api.github.com/repos/testproject')
- self.assertEqual(extractor.project, self.project)
+ import json
+ from StringIO import StringIO
+ self.extractor = github.GitHubProjectExtractor('testproject')
+ d = dict(description='project description',
+ homepage='http://example.com')
+ self.extractor.urlopen = lambda url: StringIO(json.dumps(d))
def test_get_summary(self):
- extractor = github.GitHubProjectExtractor(self.project, 'testproject', 'project_info')
- extractor.page = {'description': 'test summary'}
- extractor.get_summmary()
- self.assertEqual(self.project.summary, 'test summary')
+ self.assertEqual(self.extractor.get_summary(), 'project description')
+
+ def test_get_homepage(self):
+ self.assertEqual(self.extractor.get_homepage(), 'http://example.com')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/tests/github/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_tasks.py b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
index b478bc9..86b321e 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_tasks.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
@@ -26,6 +26,7 @@ from ...github import tasks
def test_import_project_info(c, session, ghpe):
c.project = mock.Mock(name='project')
tasks.import_project_info('my-project')
- ghpe.assert_called_once_with(c.project, 'my-project', 'project_info')
- ghpe.return_value.get_summmary.assert_called_once_with()
+ ghpe.assert_called_once_with('my-project')
+ ghpe.return_value.get_summary.assert_called_once_with()
+ ghpe.return_value.get_homepage.assert_called_once_with()
session.flush_all.assert_called_once_with()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/2d5cf6c5/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index 92c5fb2..d5a9f22 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -28,7 +28,8 @@ from forgeimporters import base
class TestGoogleCodeProjectExtractor(TestCase):
def setUp(self):
self._p_urlopen = mock.patch.object(base.ProjectExtractor, 'urlopen')
- self._p_soup = mock.patch.object(google, 'BeautifulSoup')
+ # self._p_soup = mock.patch.object(google, 'BeautifulSoup')
+ self._p_soup = mock.patch.object(base, 'BeautifulSoup')
self.urlopen = self._p_urlopen.start()
self.soup = self._p_soup.start()
self.project = mock.Mock(name='project')
[3/7] git commit: [#6531] ticket:413 Remove github username from
project name
Posted by jo...@apache.org.
[#6531] ticket:413 Remove github username from project name
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/44762cbc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/44762cbc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/44762cbc
Branch: refs/heads/master
Commit: 44762cbc2a021ac36fdde1bbee3f04e67785fd5f
Parents: 69db59c
Author: Yuriy Arhipov <yu...@yandex.ru>
Authored: Thu Aug 22 15:22:55 2013 +0400
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Aug 26 17:19:57 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/github/project.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/44762cbc/ForgeImporters/forgeimporters/github/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/project.py b/ForgeImporters/forgeimporters/github/project.py
index 4af2bdf..58485a7 100644
--- a/ForgeImporters/forgeimporters/github/project.py
+++ b/ForgeImporters/forgeimporters/github/project.py
@@ -45,7 +45,8 @@ class GitHubProjectImporter(base.ProjectImporter):
def after_project_create(self, project, **kw):
project.set_tool_data('github', project_name=project.name)
- tasks.import_project_info.post(project.name)
+ project_name = '%s/%s' % (kw['user_name'], kw['project_name'])
+ tasks.import_project_info.post(project_name)
@with_trailing_slash
@expose(index_template)
@@ -56,7 +57,6 @@ class GitHubProjectImporter(base.ProjectImporter):
@expose()
@validate(process_validator)
def process(self, **kw):
- kw['project_name'] = '%s/%s' % (kw['user_name'], kw['project_name'])
kw['tools'] = ''
return super(self.__class__, self).process(**kw)
[7/7] git commit: [#6531] Added timeout param to urlopen and fixed
retries count logging
Posted by jo...@apache.org.
[#6531] Added timeout param to urlopen and fixed retries count logging
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/8c6605d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/8c6605d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/8c6605d3
Branch: refs/heads/master
Commit: 8c6605d3f32a10b6bd2bd5c19834064c0cdbae34
Parents: 0688afc
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Mon Aug 26 19:09:10 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Aug 26 19:09:10 2013 +0000
----------------------------------------------------------------------
Allura/allura/lib/helpers.py | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/8c6605d3/Allura/allura/lib/helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index 8095f79..e545289 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -932,7 +932,7 @@ class exceptionless(object):
return inner
-def urlopen(url, retries=3, codes=(408,)):
+def urlopen(url, retries=3, codes=(408,), timeout=None):
"""Open url, optionally retrying if an error is encountered.
Socket timeouts will always be retried if retries > 0.
@@ -942,20 +942,23 @@ def urlopen(url, retries=3, codes=(408,)):
:param codes: HTTP error codes that should be retried.
"""
+ attempts = 0
while True:
try:
- return urllib2.urlopen(url)
+ return urllib2.urlopen(url, timeout=timeout)
except (urllib2.HTTPError, socket.timeout) as e:
- if retries and (isinstance(e, socket.timeout) or
+ attempts += 1
+ if attempts < retries and (isinstance(e, socket.timeout) or
e.code in codes):
- retries -= 1
continue
else:
try:
url_string = url.get_full_url() # if url is Request obj
except Exception:
url_string = url
- log.exception('Failed after %s retries on url: %s: %s', retries, url_string, e)
+ if timeout is None:
+ timeout = socket.getdefaulttimeout()
+ log.exception('Failed after %s retries on url with a timeout of %s: %s: %s', attempts, timeout, url_string, e)
raise e
[2/7] git commit: [#6531] Refactored get_page to accept parser
argument
Posted by jo...@apache.org.
[#6531] Refactored get_page to accept parser argument
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/ff5af166
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/ff5af166
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/ff5af166
Branch: refs/heads/master
Commit: ff5af166fae48eaa3f0f8c7a2f969b0764d5e4e8
Parents: 2d5cf6c
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Fri Aug 23 20:51:59 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Aug 26 17:19:57 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/base.py | 13 +++-
.../forgeimporters/google/__init__.py | 65 +++++++++++---------
.../tests/google/functional/test_tracker.py | 2 +-
.../tests/google/test_extractor.py | 29 +++++----
4 files changed, 65 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ff5af166/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 3cf6774..19a3d70 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -86,12 +86,16 @@ class ProjectExtractor(object):
req.add_header('User-Agent', 'Allura Data Importer (http://sf.net/p/allura)')
return h.urlopen(req, retries=retries, codes=codes)
- def get_page(self, page_name_or_url, **kw):
+ def get_page(self, page_name_or_url, parser=None, **kw):
"""Return a Beautiful soup object for the given page name or url.
If a page name is provided, the associated url is looked up in
:attr:`PAGE_MAP`.
+ If provided, the class or callable passed in :param:`parser` will be
+ used to transform the result of the `urlopen` before returning it.
+ Otherwise, the class's :meth:`parse_page` will be used.
+
Results are cached so that subsequent calls for the same page name or
url will return the cached result rather than making another HTTP
request.
@@ -104,8 +108,10 @@ class ProjectExtractor(object):
if self.url in self._page_cache:
self.page = self._page_cache[self.url]
else:
+ if parser is None:
+ parser = self.parse_page
self.page = self._page_cache[self.url] = \
- self.parse_page(self.urlopen(self.url))
+ parser(self.urlopen(self.url))
return self.page
def get_page_url(self, page_name, **kw):
@@ -125,7 +131,8 @@ class ProjectExtractor(object):
the html.
Subclasses can override to change the behavior or handle other types
- of content (like JSON).
+ of content (like JSON). The parser can also be overridden via the
+ `parser` parameter to :meth:`get_page`
:param page: A file-like object return from :meth:`urlopen`
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ff5af166/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 29e5011..849f924 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -51,6 +51,24 @@ def _as_text(node, chunks=None):
_as_text(n, chunks)
return ''.join(chunks)
+def csv_parser(page):
+ lines = page.readlines()
+ if not lines:
+ return []
+ # skip CSV header
+ lines = lines[1:]
+ # skip "next page here" info footer
+ if not lines[-1].startswith('"'):
+ lines.pop()
+ # remove CSV wrapping (quotes, commas, newlines)
+ return [line.strip('",\n') for line in lines]
+
+def stringio_parser(page):
+ return {
+ 'content-type': page.info()['content-type'],
+ 'data': StringIO(page.read()),
+ }
+
class GoogleCodeProjectExtractor(ProjectExtractor):
BASE_URL = 'http://code.google.com'
@@ -88,11 +106,9 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
if icon_url == self.DEFAULT_ICON:
return
icon_name = urllib.unquote(urlparse(icon_url).path).split('/')[-1]
- fp_ish = self.urlopen(icon_url)
- fp = StringIO(fp_ish.read())
+ icon = File(icon_url, icon_name)
M.ProjectFile.save_image(
- icon_name, fp,
- fp_ish.info()['content-type'].split(';')[0], # strip off charset=x extra param,
+ icon_name, icon.file, icon.type,
square=True, thumbnail_size=(48,48),
thumbnail_meta={'project_id': project._id, 'category': 'icon'})
@@ -115,16 +131,6 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
raise Exception("Unknown repo type: {0}".format(repo_type.text))
@classmethod
- def _get_issue_ids_page(cls, project_name, start):
- url = cls.PAGE_MAP['issues_csv'].format(project_name=project_name, start=start)
- with closing(cls.urlopen(url)) as fp:
- lines = fp.readlines()[1:] # skip CSV header
- if not lines[-1].startswith('"'):
- lines.pop() # skip "next page here" info footer
- issue_ids = [line.strip('",\n') for line in lines]
- return issue_ids
-
- @classmethod
def iter_issues(cls, project_name):
"""
Iterate over all issues for a project,
@@ -133,13 +139,14 @@ class GoogleCodeProjectExtractor(ProjectExtractor):
start = 0
limit = 100
- while True:
- issue_ids = cls._get_issue_ids_page(project_name, start)
- if len(issue_ids) <= 0:
+ extractor = cls(project_name, 'issues_csv', parser=csv_parser, start=start)
+ while extractor.page:
+ if len(extractor.page) <= 0:
return
- for issue_id in issue_ids:
+ for issue_id in extractor.page:
yield (int(issue_id), cls(project_name, 'issue', issue_id=issue_id))
start += limit
+ extractor.get_page('issues_csv', parser=csv_parser, start=start)
def get_issue_summary(self):
text = self.page.find(id='issueheader').findAll('td', limit=2)[1].span.string.strip()
@@ -256,14 +263,16 @@ class Comment(object):
)
return text
-class Attachment(object):
- def __init__(self, tag):
- self.url = urljoin(GoogleCodeProjectExtractor.BASE_URL, tag.get('href'))
- self.filename = parse_qs(urlparse(self.url).query)['name'][0]
- self.type = None
+class File(object):
+ def __init__(self, url, filename):
+ extractor = GoogleCodeProjectExtractor(None, url, parser=stringio_parser)
+ self.url = url
+ self.filename = filename
+ self.type = extractor.page['content-type'].split(';')[0]
+ self.file = extractor.page['data']
- @property
- def file(self):
- fp_ish = GoogleCodeProjectExtractor(None).urlopen(self.url)
- fp = StringIO(fp_ish.read())
- return fp
+class Attachment(File):
+ def __init__(self, tag):
+ url = urljoin(GoogleCodeProjectExtractor.BASE_URL, tag.get('href'))
+ filename = parse_qs(urlparse(url).query)['name'][0]
+ super(Attachment, self).__init__(url, filename)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ff5af166/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py b/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
index 184f7fd..2e5f542 100644
--- a/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/google/functional/test_tracker.py
@@ -51,7 +51,7 @@ class TestGCTrackerImporter(TestCase):
with mock.patch.object(base.h, 'urlopen') as urlopen,\
mock.patch.object(google.tracker, 'GoogleCodeProjectExtractor') as GPE,\
mock.patch('forgetracker.tasks.update_bin_counts') as ubc:
- urlopen.side_effect = lambda req, **kw: mock.Mock(read=req.get_full_url)
+ urlopen.side_effect = lambda req, **kw: mock.Mock(read=req.get_full_url, info=lambda:{'content-type': 'text/plain'})
GPE.iter_issues.return_value = [(issue_id, issue)]
gti = google.tracker.GoogleCodeTrackerImporter()
gti.import_tool(self.project, self.user, 'test-issue-project', mount_point='test-issue')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ff5af166/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
index d5a9f22..668662e 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -64,6 +64,10 @@ class TestGoogleCodeProjectExtractor(TestCase):
page = extractor.get_page('source_browse')
self.assertEqual(2, self.urlopen.call_count)
self.assertEqual(page, extractor._page_cache['http://code.google.com/p/my-project/source/browse/'])
+ parser = mock.Mock(return_value='parsed')
+ page = extractor.get_page('url', parser=parser)
+ self.assertEqual(page, 'parsed')
+ self.assertEqual(page, extractor._page_cache['url'])
def test_get_page_url(self):
extractor = google.GoogleCodeProjectExtractor('my-project')
@@ -79,22 +83,20 @@ class TestGoogleCodeProjectExtractor(TestCase):
extractor.page.find.assert_called_once_with(itemprop='description')
self.assertEqual(self.project.short_description, 'My Super Project')
- @mock.patch.object(google, 'StringIO')
+ @mock.patch.object(google, 'File')
@mock.patch.object(google, 'M')
- def test_get_icon(self, M, StringIO):
- self.urlopen.return_value.info.return_value = {'content-type': 'image/png'}
+ def test_get_icon(self, M, File):
+ File.return_value.type = 'image/png'
+ File.return_value.file = 'data'
extractor = google.GoogleCodeProjectExtractor('my-project', 'project_info')
extractor.page.find.return_value.get.return_value = 'http://example.com/foo/bar/my-logo.png'
- self.urlopen.reset_mock()
extractor.get_icon(self.project)
extractor.page.find.assert_called_once_with(itemprop='image')
- self.urlopen.assert_called_once_with('http://example.com/foo/bar/my-logo.png')
- self.urlopen.return_value.info.assert_called_once_with()
- StringIO.assert_called_once_with(self.urlopen.return_value.read.return_value)
+ File.assert_called_once_with('http://example.com/foo/bar/my-logo.png', 'my-logo.png')
M.ProjectFile.save_image.assert_called_once_with(
- 'my-logo.png', StringIO.return_value, 'image/png', square=True,
+ 'my-logo.png', 'data', 'image/png', square=True,
thumbnail_size=(48,48), thumbnail_meta={
'project_id': self.project._id, 'category': 'icon'})
@@ -209,19 +211,22 @@ class TestGoogleCodeProjectExtractor(TestCase):
'OpSys-OSX',
])
- def test_get_issue_attachments(self):
+ @mock.patch.object(google, 'StringIO')
+ def test_get_issue_attachments(self, StringIO):
+ self.urlopen.return_value.info.return_value = {'content-type': 'text/plain; foo'}
test_issue = open(pkg_resources.resource_filename('forgeimporters', 'tests/data/google/test-issue.html')).read()
gpe = self._make_extractor(test_issue)
attachments = gpe.get_issue_attachments()
self.assertEqual(len(attachments), 2)
self.assertEqual(attachments[0].filename, 'at1.txt')
self.assertEqual(attachments[0].url, 'http://allura-google-importer.googlecode.com/issues/attachment?aid=70000000&name=at1.txt&token=3REU1M3JUUMt0rJUg7ldcELt6LA%3A1376059941255')
- self.assertIsNone(attachments[0].type)
+ self.assertEqual(attachments[0].type, 'text/plain')
self.assertEqual(attachments[1].filename, 'at2.txt')
self.assertEqual(attachments[1].url, 'http://allura-google-importer.googlecode.com/issues/attachment?aid=70000001&name=at2.txt&token=C9Hn4s1-g38hlSggRGo65VZM1ys%3A1376059941255')
- self.assertIsNone(attachments[1].type)
+ self.assertEqual(attachments[1].type, 'text/plain')
- def test_iter_comments(self):
+ @mock.patch.object(google, 'StringIO')
+ def test_iter_comments(self, StringIO):
test_issue = open(pkg_resources.resource_filename('forgeimporters', 'tests/data/google/test-issue.html')).read()
gpe = self._make_extractor(test_issue)
comments = list(gpe.iter_comments())
[5/7] git commit: [#6531] ticket:413 homepage import from allura
Posted by jo...@apache.org.
[#6531] ticket:413 homepage import from allura
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/0008c42b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/0008c42b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/0008c42b
Branch: refs/heads/master
Commit: 0008c42bf0614433d2baa18a2c5b082233b76416
Parents: 44762cb
Author: Anton Kasyanov <mi...@gmail.com>
Authored: Thu Aug 22 18:05:09 2013 +0300
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Aug 26 17:19:57 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/github/__init__.py | 3 +++
ForgeImporters/forgeimporters/github/tasks.py | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0008c42b/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 472429e..1ce4398 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -41,3 +41,6 @@ class GitHubProjectExtractor(object):
def get_summmary(self):
self.project.summary = self.page['description']
+
+ def get_homepage(self):
+ self.project.external_homepage = self.page['homepage']
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0008c42b/ForgeImporters/forgeimporters/github/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tasks.py b/ForgeImporters/forgeimporters/github/tasks.py
index 4fd928c..be2325d 100644
--- a/ForgeImporters/forgeimporters/github/tasks.py
+++ b/ForgeImporters/forgeimporters/github/tasks.py
@@ -28,4 +28,5 @@ from . import GitHubProjectExtractor
def import_project_info(project_name):
extractor = GitHubProjectExtractor(c.project, project_name, 'project_info')
extractor.get_summmary()
+ extractor.get_homepage()
ThreadLocalORMSession.flush_all()