You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/07/23 00:54:14 UTC
[07/15] git commit: [#6456] Added Google Code project importer
[#6456] Added Google Code project importer
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/b82a3767
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/b82a3767
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/b82a3767
Branch: refs/heads/cj/6456
Commit: b82a37670d20fb55fe0848db454bc180d901128b
Parents: d971b6c
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Fri Jul 19 19:21:05 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Mon Jul 22 18:36:15 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/google/__init__.py | 17 +--
ForgeImporters/forgeimporters/google/project.py | 83 +++++++++++++++
ForgeImporters/forgeimporters/google/tasks.py | 33 ++++++
.../google/templates/project.html | 104 +++++++++++++++++++
.../forgeimporters/tests/google/__init__.py | 0
.../tests/google/test_extractor.py | 93 +++++++++++++++++
.../forgeimporters/tests/google/test_tasks.py | 33 ++++++
ForgeImporters/setup.py | 1 +
8 files changed, 358 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/__init__.py b/ForgeImporters/forgeimporters/google/__init__.py
index 0feb64a..e01dcfb 100644
--- a/ForgeImporters/forgeimporters/google/__init__.py
+++ b/ForgeImporters/forgeimporters/google/__init__.py
@@ -19,6 +19,10 @@ import urllib
import urllib2
from urlparse import urlparse
from collections import defaultdict
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
from BeautifulSoup import BeautifulSoup
@@ -50,19 +54,20 @@ class GoogleCodeProjectExtractor(object):
self.page = BeautifulSoup(page)
def get_short_description(self):
- self.project.short_description = str(self.page.find(itemprop='description')).strip()
+ self.project.short_description = self.page.find(itemprop='description').string.strip()
def get_icon(self):
- icon_url = self.page.find(itemprop='image').src
+ icon_url = self.page.find(itemprop='image').attrMap['src']
icon_name = urllib.unquote(urlparse(icon_url).path).split('/')[-1]
- fp = urllib2.urlopen(icon_url)
+ fp_ish = urllib2.urlopen(icon_url)
+ fp = StringIO(fp_ish.read())
M.ProjectFile.save_image(
- icon_name, fp, fp.info()['content-type'], square=True,
+ icon_name, fp, fp_ish.info()['content-type'], square=True,
thumbnail_size=(48,48),
- thumbnail_meta=dict(project_id=self.project._id, category='icon'))
+ thumbnail_meta={'project_id': self.project._id, 'category': 'icon'})
def get_license(self):
- license = str(self.page.find(text='Code license:').findNext('td')).strip()
+ license = self.page.find(text='Code license').findNext().find('a').string.strip()
trove = M.TroveCategory.query.get(fullname=self.LICENSE_MAP[license])
self.project.trove_license.append(trove._id)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/project.py b/ForgeImporters/forgeimporters/google/project.py
new file mode 100644
index 0000000..fdd3c97
--- /dev/null
+++ b/ForgeImporters/forgeimporters/google/project.py
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from tg import expose, validate, flash, redirect
+from tg.decorators import with_trailing_slash
+from pylons import tmpl_context as c
+from formencode import validators as fev, schema
+
+from allura import model as M
+from allura.lib.decorators import require_post
+from allura.lib.widgets.forms import NeighborhoodProjectShortNameValidator
+from allura.lib import helpers as h
+
+from .. import base
+from . import tasks
+
+
+
+class GoogleCodeProjectForm(schema.Schema):
+ neighborhood = fev.PlainText(not_empty=True)
+ gc_project_name = fev.Regex(r'^[a-z0-9][a-z0-9-]{,61}$', not_empty=True)
+ project_unixname = NeighborhoodProjectShortNameValidator()
+ tools = base.ToolsValidator('Google Code')
+
+
+class GoogleCodeProjectImporter(base.ProjectImporter):
+ source = 'Google Code'
+
+ @with_trailing_slash
+ @expose('jinja:forgeimporters.google:templates/project.html')
+ def index(self, **kw):
+ neighborhood = M.Neighborhood.query.get(url_prefix='/p/')
+ return {'importer': self, 'neighborhood': neighborhood}
+
+ @require_post()
+ @expose()
+ @validate(GoogleCodeProjectForm(), error_handler=index)
+ def process(self, gc_project_name=None, project_unixname=None, tools=None, **kw):
+ neighborhood = M.Neighborhood.query.get(url_prefix='/p/')
+ project_name = h.really_unicode(gc_project_name).encode('utf-8')
+ project_unixname = h.really_unicode(project_unixname).encode('utf-8').lower()
+
+ try:
+ c.project = neighborhood.register_project(project_unixname,
+ project_name=project_name)
+ except exceptions.ProjectOverlimitError:
+ flash("You have exceeded the maximum number of projects you are allowed to create", 'error')
+ redirect('.')
+ except exceptions.ProjectRatelimitError:
+ flash("Project creation rate limit exceeded. Please try again later.", 'error')
+ redirect('.')
+ except Exception as e:
+ log.error('error registering project: %s', project_unixname, exc_info=True)
+ flash('Internal Error. Please try again later.', 'error')
+ redirect('.')
+
+ c.project.set_tool_data('google-code', project_name=gc_project_name)
+ tasks.import_project_info.post()
+ for tool in tools:
+ tool.import_tool(c.project)
+
+ flash('Welcome to the SourceForge Project System! '
+ 'Your project data will be imported and should show up here shortly.')
+ redirect(c.project.script_name + 'admin/overview')
+
+ @expose('json:')
+ @validate(GoogleCodeProjectForm())
+ def check_names(self, gc_project_name=None, project_unixname=None, tools=None, **kw):
+ return c.form_errors
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tasks.py b/ForgeImporters/forgeimporters/google/tasks.py
new file mode 100644
index 0000000..79851c3
--- /dev/null
+++ b/ForgeImporters/forgeimporters/google/tasks.py
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pylons import tmpl_context as c
+
+from ming.orm import ThreadLocalORMSession
+
+from allura.lib.decorators import task
+
+from . import GoogleCodeProjectExtractor
+
+
+@task
+def import_project_info():
+ extractor = GoogleCodeProjectExtractor(c.project, 'project_info')
+ extractor.get_short_description()
+ extractor.get_icon()
+ extractor.get_license()
+ ThreadLocalORMSession.flush_all()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/google/templates/project.html
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/templates/project.html b/ForgeImporters/forgeimporters/google/templates/project.html
new file mode 100644
index 0000000..ae9362a
--- /dev/null
+++ b/ForgeImporters/forgeimporters/google/templates/project.html
@@ -0,0 +1,104 @@
+{#-
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
+{% extends 'forgeimporters:templates/project_base.html' %}
+
+{% block extra_css %}
+ {{ super() }}
+ <style type="text/css">
+ #project-import-form #project-fields input {
+ width: 88%;
+ }
+
+ .hidden { display: none; }
+ </style>
+{% endblock %}
+
+{% block extra_js %}
+ {{ super() }}
+ <script type="text/javascript">
+ var timers = {};
+ function delay(callback, ms) {
+ clearTimeout(timers[callback]);
+ timers[callback] = setTimeout(callback, ms);
+ }
+
+ var manual = false;
+ function suggest_name() {
+ if (!manual) {
+ $('#project_unixname').val($('#gc_project_name').val()).trigger('change');
+ }
+ }
+
+ function check_names() {
+ var data = {
+ 'neighborhood': $('#neighborhood').val(),
+ 'gc_project_name': $('#gc_project_name').val(),
+ 'project_unixname': $('#project_unixname').val()
+ };
+ $.getJSON('check_names', data, function(result) {
+ $('#gc_project_name_error').addClass('hidden');
+ $('#project_unixname_error').addClass('hidden');
+ for(var field in result) {
+ $('#'+field+'_error').text(result[field]).removeClass('hidden');
+ }
+ });
+ }
+
+ function update_url() {
+ $('#url-fragment').text($('#project_unixname').val());
+ }
+
+ $(function() {
+ $('#gc_project_name').focus().bind('change keyup', suggest_name);
+
+ $('#project_unixname').bind('change keyup', function(event) {
+ if (event.type == 'keyup') {
+ manual = true;
+ }
+ update_url();
+ delay(check_names, 500);
+ });
+ });
+ </script>
+{% endblock %}
+
+{% block project_fields %}
+ <div class="grid-6">
+ <label>Google Project Name</label>
+ </div>
+ <div class="grid-10">
+ <input id="gc_project_name" name="gc_project_name" value="{{c.form_values['gc_project_name']}}"/>
+ <div id="gc_project_name_error" class="error{% if not c.form_errors['gc_project_name'] %} hidden{% endif %}">
+ {{c.form_errors['gc_project_name']}}
+ </div>
+ </div>
+
+ <div class="grid-6" style="clear:left">
+ <label>URL Name</label>
+ </div>
+ <div class="grid-10">
+ <input id="project_unixname" name="project_unixname" value="{{c.form_values['project_unixname']}}"/>
+ <div id="project_unixname_error" class="error{% if not c.form_errors['project_unixname'] %} hidden{% endif %}">
+ {{c.form_errors['project_unixname']}}
+ </div>
+ <div id="project-url">
+ http://{{request.environ['HTTP_HOST']}}{{neighborhood.url()}}<span id="url-fragment">{{c.form_values['project_unixname']}}</span>
+ </div>
+ </div>
+{% endblock %}
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/tests/google/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/__init__.py b/ForgeImporters/forgeimporters/tests/google/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/tests/google/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_extractor.py b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
new file mode 100644
index 0000000..f03c82c
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/google/test_extractor.py
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from unittest import TestCase
+
+import mock
+
+from ... import google
+
+
+class TestGoogleCodeProjectExtractor(TestCase):
+ def setUp(self):
+ self._p_urlopen = mock.patch.object(google.urllib2, 'urlopen')
+ self._p_soup = mock.patch.object(google, 'BeautifulSoup')
+ self.urlopen = self._p_urlopen.start()
+ self.soup = self._p_soup.start()
+ self.project = mock.Mock(name='project')
+ self.project.get_tool_data.return_value = 'my-project'
+
+ def tearDown(self):
+ self._p_urlopen.stop()
+ self._p_soup.stop()
+
+ def test_init(self):
+ extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+
+ self.project.get_tool_data.assert_called_once_with('google-code', 'project_name')
+ self.urlopen.assert_called_once_with('http://code.google.com/p/my-project/')
+ self.assertEqual(extractor.project, self.project)
+ self.soup.assert_called_once_with(self.urlopen.return_value)
+ self.assertEqual(extractor.page, self.soup.return_value)
+
+ def test_get_short_description(self):
+ extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+ extractor.page.find.return_value.string = 'My Super Project'
+
+ extractor.get_short_description()
+
+ extractor.page.find.assert_called_once_with(itemprop='description')
+ self.assertEqual(self.project.short_description, 'My Super Project')
+
+ @mock.patch.object(google, 'StringIO')
+ @mock.patch.object(google, 'M')
+ def test_get_icon(self, M, StringIO):
+ self.urlopen.return_value.info.return_value = {'content-type': 'image/png'}
+ extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+ extractor.page.find.return_value.attrMap = {'src': 'http://example.com/foo/bar/my-logo.png'}
+ self.urlopen.reset_mock()
+
+ extractor.get_icon()
+
+ extractor.page.find.assert_called_once_with(itemprop='image')
+ self.urlopen.assert_called_once_with('http://example.com/foo/bar/my-logo.png')
+ self.urlopen.return_value.info.assert_called_once_with()
+ StringIO.assert_called_once_with(self.urlopen.return_value.read.return_value)
+ M.ProjectFile.save_image.assert_called_once_with(
+ 'my-logo.png', StringIO.return_value, 'image/png', square=True,
+ thumbnail_size=(48,48), thumbnail_meta={
+ 'project_id': self.project._id, 'category': 'icon'})
+
+ @mock.patch.object(google, 'M')
+ def test_get_license(self, M):
+ self.project.trove_license = []
+ extractor = google.GoogleCodeProjectExtractor(self.project, 'project_info')
+ extractor.page.find.return_value.findNext.return_value.find.return_value.string = ' New BSD License '
+ trove = M.TroveCategory.query.get.return_value
+
+ extractor.get_license()
+
+ extractor.page.find.assert_called_once_with(text='Code license')
+ extractor.page.find.return_value.findNext.assert_called_once_with()
+ extractor.page.find.return_value.findNext.return_value.find.assert_called_once_with('a')
+ self.assertEqual(self.project.trove_license, [trove._id])
+ M.TroveCategory.query.get.assert_called_once_with(fullname='BSD License')
+
+ M.TroveCategory.query.get.reset_mock()
+ extractor.page.find.return_value.findNext.return_value.find.return_value.string = 'non-existant license'
+ extractor.get_license()
+ M.TroveCategory.query.get.assert_called_once_with(fullname='Other/Proprietary License')
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/forgeimporters/tests/google/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tasks.py b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
new file mode 100644
index 0000000..a21d083
--- /dev/null
+++ b/ForgeImporters/forgeimporters/tests/google/test_tasks.py
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import mock
+
+from ...google import tasks
+
+
+@mock.patch.object(tasks, 'GoogleCodeProjectExtractor')
+@mock.patch.object(tasks, 'ThreadLocalORMSession')
+@mock.patch.object(tasks, 'c')
+def test_import_project_info(c, session, gpe):
+ c.project = mock.Mock(name='project')
+ tasks.import_project_info()
+ gpe.assert_called_once_with(c.project, 'project_info')
+ gpe.return_value.get_short_description.assert_called_once_with()
+ gpe.return_value.get_icon.assert_called_once_with()
+ gpe.return_value.get_license.assert_called_once_with()
+ session.flush_all.assert_called_once_with()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/b82a3767/ForgeImporters/setup.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/setup.py b/ForgeImporters/setup.py
index 5e45638..a928504 100644
--- a/ForgeImporters/setup.py
+++ b/ForgeImporters/setup.py
@@ -34,6 +34,7 @@ setup(name='ForgeImporters',
entry_points="""
# -*- Entry points: -*-
[allura.project_importers]
+ google-code = forgeimporters.google.project:GoogleCodeProjectImporter
[allura.importers]
""",)