You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/11/13 23:44:57 UTC
[1/8] git commit: [#6656] ticket:437 More careful rate limit checking
Updated Branches:
refs/heads/master 2c6339ab4 -> c1fc7144e
[#6656] ticket:437 More careful rate limit checking
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3c07c836
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3c07c836
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3c07c836
Branch: refs/heads/master
Commit: 3c07c83643389b4402fb2c9b830ce7f835114517
Parents: 4d518a0
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 16:14:50 2013 +0300
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:43:45 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/github/__init__.py | 32 +++++++++++++-----
.../tests/github/test_extractor.py | 35 +++++++++++++++++---
2 files changed, 54 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c07c836/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 4932697..951d670 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -19,6 +19,7 @@ import re
import logging
import json
import time
+import urllib2
from datetime import datetime
from forgeimporters import base
@@ -49,18 +50,31 @@ class GitHubProjectExtractor(base.ProjectExtractor):
url += glue + 'access_token=' + self.token
return url
+ def wait_for_limit_reset(self, headers):
+ reset = headers.get('X-RateLimit-Reset')
+ limit = headers.get('X-RateLimit-Limit')
+ reset = datetime.utcfromtimestamp(int(reset))
+ now = datetime.utcnow()
+ log.warn('Rate limit exceeded (%s requests/hour). '
+ 'Sleeping until %s UTC' % (limit, reset))
+ time.sleep((reset - now).total_seconds())
+
def urlopen(self, url, **kw):
- resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+ try:
+ resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+ except urllib2.HTTPError as e:
+ # GitHub will return 403 if rate limit exceeded.
+ # We're checking for limit on every request below, but we still
+ # can get 403 if other import task exceeds the limit before.
+ if e.code == 403 and e.info().get('X-RateLimit-Remaining') == '0':
+ self.wait_for_limit_reset(e.info())
+ return self.urlopen(url, **kw)
+ else:
+ raise e
remain = resp.info().get('X-RateLimit-Remaining')
if remain and int(remain) == 0:
- reset = resp.info().get('X-RateLimit-Reset')
- limit = resp.info().get('X-RateLimit-Limit')
- reset = datetime.utcfromtimestamp(int(reset))
- now = datetime.utcnow()
- log.warn('Rate limit exceeded (%s requests/hour). '
- 'Sleeping until %s UTC' % (limit, reset))
- time.sleep((reset - now).total_seconds())
- self.urlopen(url, **kw)
+ self.wait_for_limit_reset(resp.info())
+ return self.urlopen(url, **kw)
return resp
def get_next_page_url(self, link):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c07c836/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index b3ad2f2..e6c96e3 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -16,8 +16,8 @@
# under the License.
import json
-from datetime import datetime
from unittest import TestCase
+import urllib2
from mock import patch, Mock
@@ -164,8 +164,7 @@ class TestGitHubProjectExtractor(TestCase):
response_limit_exceeded.info = lambda: limit_exceeded_headers
response_ok = StringIO('{}')
response_ok.info = lambda: {}
- results = [response_limit_exceeded, response_ok]
- urlopen.side_effect = lambda *a, **kw: results.pop(0)
+ urlopen.side_effect = [response_limit_exceeded, response_ok]
e = github.GitHubProjectExtractor('test_project')
e.get_page('fake')
self.assertEqual(sleep.call_count, 1)
@@ -175,8 +174,36 @@ class TestGitHubProjectExtractor(TestCase):
'Sleeping until 2013-10-25 09:32:02 UTC'
)
sleep.reset_mock(); urlopen.reset_mock(); log.warn.reset_mock()
- urlopen.side_effect = lambda *a, **kw: response_ok
+ response_ok = StringIO('{}')
+ response_ok.info = lambda: {}
+ urlopen.side_effect = [response_ok]
e.get_page('fake 2')
self.assertEqual(sleep.call_count, 0)
self.assertEqual(urlopen.call_count, 1)
self.assertEqual(log.warn.call_count, 0)
+
+ @patch('forgeimporters.base.h.urlopen')
+ @patch('forgeimporters.github.time.sleep')
+ @patch('forgeimporters.github.log')
+ def test_urlopen_rate_limit_403(self, log, sleep, urlopen):
+ '''Test that urlopen catches 403 which may happen if limit exceeded by another task'''
+ limit_exceeded_headers = {
+ 'X-RateLimit-Limit': '10',
+ 'X-RateLimit-Remaining': '0',
+ 'X-RateLimit-Reset': '1382693522',
+ }
+ def urlopen_side_effect(*a, **kw):
+ mock_resp = StringIO('{}')
+ mock_resp.info = lambda: {}
+ urlopen.side_effect = [mock_resp]
+ raise urllib2.HTTPError(
+ 'url', 403, 'msg', limit_exceeded_headers, StringIO('{}'))
+ urlopen.side_effect = urlopen_side_effect
+ e = github.GitHubProjectExtractor('test_project')
+ e.get_page('fake')
+ self.assertEqual(sleep.call_count, 1)
+ self.assertEqual(urlopen.call_count, 2)
+ log.warn.assert_called_once_with(
+ 'Rate limit exceeded (10 requests/hour). '
+ 'Sleeping until 2013-10-25 09:32:02 UTC'
+ )
[5/8] git commit: [#6656] ticket:437 Use token in github importers
Posted by jo...@apache.org.
[#6656] ticket:437 Use token in github importers
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/0775036a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/0775036a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/0775036a
Branch: refs/heads/master
Commit: 0775036a41f7bc3e6616cbc09b7093d18bf16570
Parents: d1f23ad
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 15:28:53 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:45:41 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/github/__init__.py | 2 +-
ForgeImporters/forgeimporters/github/code.py | 2 +-
ForgeImporters/forgeimporters/github/tasks.py | 2 +-
ForgeImporters/forgeimporters/github/tracker.py | 2 +-
ForgeImporters/forgeimporters/github/wiki.py | 2 +-
5 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0775036a/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index a92f4a1..fb34e6b 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -46,7 +46,7 @@ class GitHubProjectExtractor(base.ProjectExtractor):
self.token = None
user = kw.pop('user', None)
if user:
- self.token = user.get_tool_data('GitHubProjectExtractor', 'token')
+ self.token = user.get_tool_data('GitHubProjectImport', 'token')
super(GitHubProjectExtractor, self).__init__(*args, **kw)
def add_token(self, url):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0775036a/ForgeImporters/forgeimporters/github/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/code.py b/ForgeImporters/forgeimporters/github/code.py
index 1017eba..4ce2183 100644
--- a/ForgeImporters/forgeimporters/github/code.py
+++ b/ForgeImporters/forgeimporters/github/code.py
@@ -92,7 +92,7 @@ class GitHubRepoImporter(ToolImporter):
"""
project_name = "%s/%s" % (user_name, project_name)
- extractor = GitHubProjectExtractor(project_name)
+ extractor = GitHubProjectExtractor(project_name, user=user)
repo_url = extractor.get_repo_url()
app = project.install_app(
"Git",
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0775036a/ForgeImporters/forgeimporters/github/tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tasks.py b/ForgeImporters/forgeimporters/github/tasks.py
index 9abdb3d..d74a9cd 100644
--- a/ForgeImporters/forgeimporters/github/tasks.py
+++ b/ForgeImporters/forgeimporters/github/tasks.py
@@ -31,7 +31,7 @@ def import_project_info(project_name):
from forgeimporters.github.project import GitHubProjectImporter
importer = GitHubProjectImporter(None)
with ImportErrorHandler(importer, project_name, c.project) as handler:
- extractor = GitHubProjectExtractor(project_name)
+ extractor = GitHubProjectExtractor(project_name, user=c.user)
c.project.summary = extractor.get_summary()
c.project.external_homepage = extractor.get_homepage()
ThreadLocalORMSession.flush_all()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0775036a/ForgeImporters/forgeimporters/github/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tracker.py b/ForgeImporters/forgeimporters/github/tracker.py
index 5e3c134..288379b 100644
--- a/ForgeImporters/forgeimporters/github/tracker.py
+++ b/ForgeImporters/forgeimporters/github/tracker.py
@@ -114,7 +114,7 @@ class GitHubTrackerImporter(ToolImporter):
self.github_markdown_converter = GitHubMarkdownConverter(
kw['user_name'], project_name)
ThreadLocalORMSession.flush_all()
- extractor = GitHubProjectExtractor(project_name)
+ extractor = GitHubProjectExtractor(project_name, user=user)
try:
M.session.artifact_orm_session._get().skip_mod_date = True
with h.push_config(c, user=M.User.anonymous(), app=app):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0775036a/ForgeImporters/forgeimporters/github/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/wiki.py b/ForgeImporters/forgeimporters/github/wiki.py
index 8fae1b1..ce3214c 100644
--- a/ForgeImporters/forgeimporters/github/wiki.py
+++ b/ForgeImporters/forgeimporters/github/wiki.py
@@ -139,7 +139,7 @@ class GitHubWikiImporter(ToolImporter):
"""
project_name = "%s/%s" % (user_name, project_name)
- extractor = GitHubProjectExtractor(project_name)
+ extractor = GitHubProjectExtractor(project_name, user=user)
if not extractor.has_wiki():
return
[3/8] git commit: [#6656] ticket:437 Use OAuth token for github api
calls
Posted by jo...@apache.org.
[#6656] ticket:437 Use OAuth token for github api calls
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d396c9e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d396c9e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d396c9e6
Branch: refs/heads/master
Commit: d396c9e6db9e45c07465d5d73375068787c3c594
Parents: 2c6339a
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 11:15:52 2013 +0300
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:43:45 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/github/__init__.py | 19 +++++++++++++--
.../tests/github/test_extractor.py | 25 +++++++++++++++++++-
2 files changed, 41 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d396c9e6/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 2963067..5726326 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -18,8 +18,6 @@
import re
import logging
import json
-import urllib
-import urllib2
from forgeimporters import base
@@ -36,6 +34,23 @@ class GitHubProjectExtractor(base.ProjectExtractor):
SUPPORTED_ISSUE_EVENTS = ('closed', 'reopened', 'assigned')
NEXT_PAGE_URL_RE = re.compile(r'<([^>]*)>; rel="next"')
+ def __init__(self, *args, **kw):
+ self.token = None
+ user = kw.pop('user', None)
+ if user:
+ self.token = user.get_tool_data('GitHubProjectExtractor', 'token')
+ super(GitHubProjectExtractor, self).__init__(*args, **kw)
+
+ def add_token(self, url):
+ if self.token:
+ glue = '&' if '?' in url else '?'
+ url += glue + 'access_token=' + self.token
+ return url
+
+ def urlopen(self, url, **kw):
+ url = self.add_token(url)
+ return super(GitHubProjectExtractor, self).urlopen(url, **kw)
+
def get_next_page_url(self, link):
if not link:
return
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d396c9e6/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index 11601a5..a26438a 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -18,6 +18,8 @@
import json
from unittest import TestCase
+from mock import patch, Mock
+
from ... import github
# Can't use cStringIO here, because we cannot set attributes or subclass it,
@@ -84,6 +86,7 @@ class TestGitHubProjectExtractor(TestCase):
def setUp(self):
self.extractor = github.GitHubProjectExtractor('test_project')
self.extractor.urlopen = self.mocked_urlopen
+
def test_get_next_page_url(self):
self.assertIsNone(self.extractor.get_next_page_url(None))
self.assertIsNone(self.extractor.get_next_page_url(''))
@@ -97,7 +100,7 @@ class TestGitHubProjectExtractor(TestCase):
link = '<https://api.github.com/repositories/8560576/issues?state=open&page=1>; rel="prev"'
self.assertIsNone(self.extractor.get_next_page_url(link))
-
+
def test_get_summary(self):
self.assertEqual(self.extractor.get_summary(), 'project description')
@@ -126,3 +129,23 @@ class TestGitHubProjectExtractor(TestCase):
def test_get_wiki_url(self):
self.assertEqual(self.extractor.get_page_url('wiki_url'), 'https://github.com/test_project.wiki')
+
+ @patch('forgeimporters.base.h.urlopen')
+ def test_urlopen(self, urlopen):
+ e = github.GitHubProjectExtractor('test_project')
+ url = 'https://github.com/u/p/'
+ e.urlopen(url)
+ request = urlopen.call_args[0][0]
+ self.assertEqual(request.get_full_url(), url)
+
+ user = Mock()
+ user.get_tool_data.return_value = 'abc'
+ e = github.GitHubProjectExtractor('test_project', user=user)
+ e.urlopen(url)
+ request = urlopen.call_args[0][0]
+ self.assertEqual(request.get_full_url(), url + '?access_token=abc')
+
+ url = 'https://github.com/u/p/?p=1'
+ e.urlopen(url)
+ request = urlopen.call_args[0][0]
+ self.assertEqual(request.get_full_url(), url + '&access_token=abc')
[2/8] git commit: [#6656] ticket:437 Check rate limit values and
sleep until reset
Posted by jo...@apache.org.
[#6656] ticket:437 Check rate limit values and sleep until reset
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/4d518a0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/4d518a0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/4d518a0f
Branch: refs/heads/master
Commit: 4d518a0f2c2e977d46f2cce6b7b81b82e6097516
Parents: d396c9e
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 12:53:52 2013 +0300
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:43:45 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/github/__init__.py | 16 ++++++++--
.../tests/github/test_extractor.py | 31 ++++++++++++++++++++
2 files changed, 45 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4d518a0f/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 5726326..4932697 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -18,6 +18,8 @@
import re
import logging
import json
+import time
+from datetime import datetime
from forgeimporters import base
@@ -48,8 +50,18 @@ class GitHubProjectExtractor(base.ProjectExtractor):
return url
def urlopen(self, url, **kw):
- url = self.add_token(url)
- return super(GitHubProjectExtractor, self).urlopen(url, **kw)
+ resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+ remain = resp.info().get('X-RateLimit-Remaining')
+ if remain and int(remain) == 0:
+ reset = resp.info().get('X-RateLimit-Reset')
+ limit = resp.info().get('X-RateLimit-Limit')
+ reset = datetime.utcfromtimestamp(int(reset))
+ now = datetime.utcnow()
+ log.warn('Rate limit exceeded (%s requests/hour). '
+ 'Sleeping until %s UTC' % (limit, reset))
+ time.sleep((reset - now).total_seconds())
+ self.urlopen(url, **kw)
+ return resp
def get_next_page_url(self, link):
if not link:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4d518a0f/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index a26438a..b3ad2f2 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -16,6 +16,7 @@
# under the License.
import json
+from datetime import datetime
from unittest import TestCase
from mock import patch, Mock
@@ -149,3 +150,33 @@ class TestGitHubProjectExtractor(TestCase):
e.urlopen(url)
request = urlopen.call_args[0][0]
self.assertEqual(request.get_full_url(), url + '&access_token=abc')
+
+ @patch('forgeimporters.base.h.urlopen')
+ @patch('forgeimporters.github.time.sleep')
+ @patch('forgeimporters.github.log')
+ def test_urlopen_rate_limit(self, log, sleep, urlopen):
+ limit_exceeded_headers = {
+ 'X-RateLimit-Limit': '10',
+ 'X-RateLimit-Remaining': '0',
+ 'X-RateLimit-Reset': '1382693522',
+ }
+ response_limit_exceeded = StringIO('{}')
+ response_limit_exceeded.info = lambda: limit_exceeded_headers
+ response_ok = StringIO('{}')
+ response_ok.info = lambda: {}
+ results = [response_limit_exceeded, response_ok]
+ urlopen.side_effect = lambda *a, **kw: results.pop(0)
+ e = github.GitHubProjectExtractor('test_project')
+ e.get_page('fake')
+ self.assertEqual(sleep.call_count, 1)
+ self.assertEqual(urlopen.call_count, 2)
+ log.warn.assert_called_once_with(
+ 'Rate limit exceeded (10 requests/hour). '
+ 'Sleeping until 2013-10-25 09:32:02 UTC'
+ )
+ sleep.reset_mock(); urlopen.reset_mock(); log.warn.reset_mock()
+ urlopen.side_effect = lambda *a, **kw: response_ok
+ e.get_page('fake 2')
+ self.assertEqual(sleep.call_count, 0)
+ self.assertEqual(urlopen.call_count, 1)
+ self.assertEqual(log.warn.call_count, 0)
[4/8] git commit: [#6656] ticket:437 GitHub OAuth flow
Posted by jo...@apache.org.
[#6656] ticket:437 GitHub OAuth flow
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/d1f23ad8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/d1f23ad8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/d1f23ad8
Branch: refs/heads/master
Commit: d1f23ad8cb3605cba94e59ad5ca6bf74d7b2fa04
Parents: 3c07c83
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 17:24:37 2013 +0300
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:43:46 2013 +0000
----------------------------------------------------------------------
Allura/development.ini | 4 ++
.../forgeimporters/github/__init__.py | 41 ++++++++++++++++++++
ForgeImporters/forgeimporters/github/project.py | 5 ++-
requirements-common.txt | 1 +
4 files changed, 50 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d1f23ad8/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 2c701cb..848955a 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -157,6 +157,10 @@ bulk_export_download_instructions = Sample instructions for {project}
importer_upload_path = /tmp/importer_upload/{nbhd}/{project}
+# GitHub importer keys
+# github_importer.client_id =
+# github_importer.client_secret =
+
# space-separated list of tool names that are valid options
# for project admins to set for their 'support_page' field
# this field is not used by default in Allura, so this option
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d1f23ad8/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 951d670..a92f4a1 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -22,6 +22,11 @@ import time
import urllib2
from datetime import datetime
+from tg import config, session, redirect, request, expose
+from tg.decorators import without_trailing_slash
+from pylons import tmpl_context as c
+from requests_oauthlib import OAuth2Session
+
from forgeimporters import base
log = logging.getLogger(__name__)
@@ -136,3 +141,39 @@ class GitHubProjectExtractor(base.ProjectExtractor):
def has_wiki(self):
return self.get_page('project_info').get('has_wiki')
+
+
+class GitHubOAuthMixin(object):
+ '''Support for github oauth web application flow.'''
+
+ def oauth_begin(self, controller_url):
+ '''controller_url is absolute url, refers the controller you mixed this class in'''
+ client_id = config.get('github_importer.client_id')
+ secret = config.get('github_importer.client_secret')
+ if not client_id or not secret:
+ return # GitHub app is not configured
+ if c.user.get_tool_data('GitHubProjectImport', 'token'):
+ return # token already exists, nothing to do
+ redirect_uri = controller_url.rstrip('/') + '/oauth_callback'
+ oauth = OAuth2Session(client_id, redirect_uri=redirect_uri)
+ auth_url, state = oauth.authorization_url('https://github.com/login/oauth/authorize')
+ session['github.oauth.state'] = state # Used in callback to prevent CSRF
+ session['github.oauth.redirect'] = controller_url
+ session.save()
+ redirect(auth_url)
+
+ @without_trailing_slash
+ @expose()
+ def oauth_callback(self, **kw):
+ client_id = config.get('github_importer.client_id')
+ secret = config.get('github_importer.client_secret')
+ if not client_id or not secret:
+ return # GitHub app is not configured
+ oauth = OAuth2Session(client_id, state=session.get('github.oauth.state'))
+ token = oauth.fetch_token(
+ 'https://github.com/login/oauth/access_token',
+ client_secret=secret,
+ authorization_response=request.url
+ )
+ c.user.set_tool_data('GitHubProjectImport', token=token['access_token'])
+ redirect(session.get('github.oauth.redirect', '/'))
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d1f23ad8/ForgeImporters/forgeimporters/github/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/project.py b/ForgeImporters/forgeimporters/github/project.py
index e04590a..29df780 100644
--- a/ForgeImporters/forgeimporters/github/project.py
+++ b/ForgeImporters/forgeimporters/github/project.py
@@ -23,9 +23,11 @@ from tg import expose, validate
from tg.decorators import with_trailing_slash
from allura.lib.decorators import require_post
+from allura.lib import helpers as h
from .. import base
from . import tasks
+from . import GitHubOAuthMixin
log = logging.getLogger(__name__)
@@ -37,7 +39,7 @@ class GitHubProjectForm(base.ProjectImportForm):
'invalid': 'Valid symbols are: letters, numbers, dashes, underscores and periods',
})
-class GitHubProjectImporter(base.ProjectImporter):
+class GitHubProjectImporter(base.ProjectImporter, GitHubOAuthMixin):
source = 'GitHub'
index_template = 'jinja:forgeimporters.github:templates/project.html'
@@ -51,6 +53,7 @@ class GitHubProjectImporter(base.ProjectImporter):
@with_trailing_slash
@expose(index_template)
def index(self, **kw):
+ self.oauth_begin(h.absurl('/p/import_project/github/'))
return super(self.__class__, self).index(**kw)
@require_post()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/d1f23ad8/requirements-common.txt
----------------------------------------------------------------------
diff --git a/requirements-common.txt b/requirements-common.txt
index 1b1d2f6..685f437 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -39,6 +39,7 @@ python-openid==2.2.5
python-oembed==0.2.1
pytidylib==0.2.1
requests==1.2.3
+requests-oauthlib==0.4.0
# for taskd proc name switching
setproctitle==1.1.7
# dep of pypeline
[6/8] git commit: [#6656] ticket:437 OAuth flow in individual github
import controllers
Posted by jo...@apache.org.
[#6656] ticket:437 OAuth flow in individual github import controllers
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/ac8fa2c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/ac8fa2c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/ac8fa2c0
Branch: refs/heads/master
Commit: ac8fa2c06b2f902b0f3a4c1a8fd8ff1b109560d6
Parents: 0775036
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 15:50:31 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:46:25 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/github/__init__.py | 7 +++----
ForgeImporters/forgeimporters/github/code.py | 6 ++++--
ForgeImporters/forgeimporters/github/project.py | 2 +-
ForgeImporters/forgeimporters/github/tracker.py | 5 +++--
ForgeImporters/forgeimporters/github/wiki.py | 5 +++--
5 files changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ac8fa2c0/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index fb34e6b..58afba3 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -146,19 +146,18 @@ class GitHubProjectExtractor(base.ProjectExtractor):
class GitHubOAuthMixin(object):
'''Support for github oauth web application flow.'''
- def oauth_begin(self, controller_url):
- '''controller_url is absolute url, refers the controller you mixed this class in'''
+ def oauth_begin(self):
client_id = config.get('github_importer.client_id')
secret = config.get('github_importer.client_secret')
if not client_id or not secret:
return # GitHub app is not configured
if c.user.get_tool_data('GitHubProjectImport', 'token'):
return # token already exists, nothing to do
- redirect_uri = controller_url.rstrip('/') + '/oauth_callback'
+ redirect_uri = request.url.rstrip('/') + '/oauth_callback'
oauth = OAuth2Session(client_id, redirect_uri=redirect_uri)
auth_url, state = oauth.authorization_url('https://github.com/login/oauth/authorize')
session['github.oauth.state'] = state # Used in callback to prevent CSRF
- session['github.oauth.redirect'] = controller_url
+ session['github.oauth.redirect'] = request.url
session.save()
redirect(auth_url)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ac8fa2c0/ForgeImporters/forgeimporters/github/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/code.py b/ForgeImporters/forgeimporters/github/code.py
index 4ce2183..6fdd0b4 100644
--- a/ForgeImporters/forgeimporters/github/code.py
+++ b/ForgeImporters/forgeimporters/github/code.py
@@ -30,6 +30,7 @@ from tg.decorators import (
)
from allura.lib.decorators import require_post
+from allura.lib import helpers as h
from allura.controllers import BaseController
from allura import model as M
@@ -39,7 +40,7 @@ from forgeimporters.base import (
ToolImporter,
ToolImportForm,
)
-from forgeimporters.github import GitHubProjectExtractor
+from forgeimporters.github import GitHubProjectExtractor, GitHubOAuthMixin
class GitHubRepoImportForm(ToolImportForm):
@@ -47,7 +48,7 @@ class GitHubRepoImportForm(ToolImportForm):
gh_user_name = fev.UnicodeString(not_empty=True)
-class GitHubRepoImportController(BaseController):
+class GitHubRepoImportController(BaseController, GitHubOAuthMixin):
def __init__(self):
self.importer = GitHubRepoImporter()
@@ -58,6 +59,7 @@ class GitHubRepoImportController(BaseController):
@with_trailing_slash
@expose('jinja:forgeimporters.github:templates/code/index.html')
def index(self, **kw):
+ self.oauth_begin()
return dict(importer=self.importer,
target_app=self.target_app)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ac8fa2c0/ForgeImporters/forgeimporters/github/project.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/project.py b/ForgeImporters/forgeimporters/github/project.py
index 29df780..2462617 100644
--- a/ForgeImporters/forgeimporters/github/project.py
+++ b/ForgeImporters/forgeimporters/github/project.py
@@ -53,7 +53,7 @@ class GitHubProjectImporter(base.ProjectImporter, GitHubOAuthMixin):
@with_trailing_slash
@expose(index_template)
def index(self, **kw):
- self.oauth_begin(h.absurl('/p/import_project/github/'))
+ self.oauth_begin()
return super(self.__class__, self).index(**kw)
@require_post()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ac8fa2c0/ForgeImporters/forgeimporters/github/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tracker.py b/ForgeImporters/forgeimporters/github/tracker.py
index 288379b..cda1466 100644
--- a/ForgeImporters/forgeimporters/github/tracker.py
+++ b/ForgeImporters/forgeimporters/github/tracker.py
@@ -44,7 +44,7 @@ from ming.orm import session, ThreadLocalORMSession
from pylons import tmpl_context as c
from pylons import app_globals as g
-from . import GitHubProjectExtractor
+from . import GitHubProjectExtractor, GitHubOAuthMixin
from ..base import ToolImporter
from forgetracker.tracker_main import ForgeTrackerApp
from forgetracker import model as TM
@@ -57,7 +57,7 @@ class GitHubTrackerImportForm(ToolImportForm):
gh_user_name = fev.UnicodeString(not_empty=True)
-class GitHubTrackerImportController(BaseController):
+class GitHubTrackerImportController(BaseController, GitHubOAuthMixin):
def __init__(self):
self.importer = GitHubTrackerImporter()
@@ -69,6 +69,7 @@ class GitHubTrackerImportController(BaseController):
@with_trailing_slash
@expose('jinja:forgeimporters.github:templates/tracker/index.html')
def index(self, **kw):
+ self.oauth_begin()
return dict(importer=self.importer,
target_app=self.target_app)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/ac8fa2c0/ForgeImporters/forgeimporters/github/wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/wiki.py b/ForgeImporters/forgeimporters/github/wiki.py
index ce3214c..33801df 100644
--- a/ForgeImporters/forgeimporters/github/wiki.py
+++ b/ForgeImporters/forgeimporters/github/wiki.py
@@ -50,7 +50,7 @@ from forgeimporters.base import (
ToolImporter,
ToolImportForm,
)
-from forgeimporters.github import GitHubProjectExtractor
+from forgeimporters.github import GitHubProjectExtractor, GitHubOAuthMixin
from forgeimporters.github.utils import GitHubMarkdownConverter
from forgewiki import model as WM
from forgewiki.converters import mediawiki2markdown
@@ -74,7 +74,7 @@ class GitHubWikiImportForm(ToolImportForm):
tool_option = fev.UnicodeString(if_missing=u'')
-class GitHubWikiImportController(BaseController):
+class GitHubWikiImportController(BaseController, GitHubOAuthMixin):
def __init__(self):
self.importer = GitHubWikiImporter()
@@ -86,6 +86,7 @@ class GitHubWikiImportController(BaseController):
@with_trailing_slash
@expose('jinja:forgeimporters.github:templates/wiki/index.html')
def index(self, **kw):
+ self.oauth_begin()
return dict(importer=self.importer,
target_app=self.target_app)
[8/8] git commit: [#6656] Updated requests version and added oauthlib
to match requests-oauthlib dependencies
Posted by jo...@apache.org.
[#6656] Updated requests version and added oauthlib to match requests-oauthlib dependencies
Signed-off-by: Cory Johns <cj...@slashdotmedia.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/c1fc7144
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/c1fc7144
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/c1fc7144
Branch: refs/heads/master
Commit: c1fc7144e9beb6c1b63434a76c2f01d5b7537384
Parents: 5f43489
Author: Cory Johns <cj...@slashdotmedia.com>
Authored: Wed Nov 13 21:37:12 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 21:37:12 2013 +0000
----------------------------------------------------------------------
requirements-common.txt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c1fc7144/requirements-common.txt
----------------------------------------------------------------------
diff --git a/requirements-common.txt b/requirements-common.txt
index 685f437..a8747b4 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -38,7 +38,8 @@ python-magic==0.4.3
python-openid==2.2.5
python-oembed==0.2.1
pytidylib==0.2.1
-requests==1.2.3
+requests==2.0.0
+oauthlib==0.4.2
requests-oauthlib==0.4.0
# for taskd proc name switching
setproctitle==1.1.7
[7/8] git commit: [#6656] ticket:437 OAuth tests
Posted by jo...@apache.org.
[#6656] ticket:437 OAuth tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/5f434893
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/5f434893
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/5f434893
Branch: refs/heads/master
Commit: 5f434893f43098a8720eb9b34c8b995da13bb72a
Parents: ac8fa2c
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 17:12:28 2013 +0000
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:47:42 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/github/tests/test_code.py | 7 ++++
.../forgeimporters/github/tests/test_tracker.py | 8 ++++
.../forgeimporters/github/tests/test_wiki.py | 7 ++++
.../tests/github/functional/test_github.py | 44 ++++++++++++++++++++
.../forgeimporters/tests/github/test_tasks.py | 3 +-
5 files changed, 68 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5f434893/ForgeImporters/forgeimporters/github/tests/test_code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tests/test_code.py b/ForgeImporters/forgeimporters/github/tests/test_code.py
index 4ad8dac..7c85058 100644
--- a/ForgeImporters/forgeimporters/github/tests/test_code.py
+++ b/ForgeImporters/forgeimporters/github/tests/test_code.py
@@ -23,6 +23,7 @@ from allura.tests import TestController
from allura.tests.decorators import with_tool
from allura import model as M
from forgeimporters.github.code import GitHubRepoImporter
+from forgeimporters.github import GitHubOAuthMixin
# important to be distinct from 'test' which ForgeGit uses, so that the tests can run in parallel and not clobber each other
@@ -105,3 +106,9 @@ class TestGitHubImportController(TestController, TestCase):
status=302).follow()
self.assertIn('Please wait and try again', r)
self.assertEqual(import_tool.post.call_count, 0)
+
+ @with_git
+ @patch.object(GitHubOAuthMixin, 'oauth_begin')
+ def test_oauth(self, oauth_begin):
+ r = self.app.get('/p/{}/admin/ext/import/github-repo/'.format(test_project_with_repo))
+ oauth_begin.assert_called_once()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5f434893/ForgeImporters/forgeimporters/github/tests/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tests/test_tracker.py b/ForgeImporters/forgeimporters/github/tests/test_tracker.py
index 938a194..d63b4d6 100644
--- a/ForgeImporters/forgeimporters/github/tests/test_tracker.py
+++ b/ForgeImporters/forgeimporters/github/tests/test_tracker.py
@@ -23,6 +23,8 @@ from allura.tests import TestController
from allura.tests.decorators import with_tool
from allura import model as M
+from forgeimporters.github import GitHubOAuthMixin
+
# important to be distinct from 'test' which ForgeTracker uses, so that the tests can run in parallel and not clobber each other
test_project_with_tracker = 'test2'
with_tracker = with_tool(test_project_with_tracker, 'tickets', 'spooky-issues', 'tickets')
@@ -69,3 +71,9 @@ class TestGitHubTrackerImportController(TestController, TestCase):
r = self.app.post(self.url + 'create', params, status=302).follow()
self.assertIn('Please wait and try again', r)
self.assertEqual(import_tool.post.call_count, 0)
+
+ @with_tracker
+ @patch.object(GitHubOAuthMixin, 'oauth_begin')
+ def test_oauth(self, oauth_begin):
+ r = self.app.get(self.url)
+ oauth_begin.assert_called_once()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5f434893/ForgeImporters/forgeimporters/github/tests/test_wiki.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tests/test_wiki.py b/ForgeImporters/forgeimporters/github/tests/test_wiki.py
index dec92a4..70ea74d 100644
--- a/ForgeImporters/forgeimporters/github/tests/test_wiki.py
+++ b/ForgeImporters/forgeimporters/github/tests/test_wiki.py
@@ -29,6 +29,7 @@ from allura.tests.decorators import with_tool, without_module
from alluratest.controller import setup_basic_test
from forgeimporters.github.wiki import GitHubWikiImporter
from forgeimporters.github.utils import GitHubMarkdownConverter
+from forgeimporters.github import GitHubOAuthMixin
# important to be distinct from 'test' which ForgeWiki uses, so that the tests can run in parallel and not clobber each other
@@ -558,3 +559,9 @@ class TestGitHubWikiImportController(TestController, TestCase):
r = self.app.post(self.url + 'create', params, status=302).follow()
self.assertIn('Please wait and try again', r)
self.assertEqual(import_tool.post.call_count, 0)
+
+ @with_wiki
+ @patch.object(GitHubOAuthMixin, 'oauth_begin')
+ def test_oauth(self, oauth_begin):
+ r = self.app.get(self.url)
+ oauth_begin.assert_called_once()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5f434893/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/functional/test_github.py b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
index afc9bd0..a9c5052 100644
--- a/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
+++ b/ForgeImporters/forgeimporters/tests/github/functional/test_github.py
@@ -15,8 +15,13 @@
# specific language governing permissions and limitations
# under the License.
+import tg
+from mock import patch, call, Mock
+from nose.tools import assert_equal
from unittest import TestCase
+
from allura.tests import TestController
+from allura import model as M
class TestGitHubImportController(TestController, TestCase):
@@ -35,3 +40,42 @@ class TestGitHubImportController(TestController, TestCase):
r = self.app.post('/p/import_project/github/process', extra_environ=dict(username='*anonymous'), status=302)
self.assertIn('/auth/', r.location)
+
+
+class TestGitHubOAuth(TestController):
+
+ def setUp(self):
+ super(TestGitHubOAuth, self).setUp()
+ tg.config['github_importer.client_id'] = 'client_id'
+ tg.config['github_importer.client_secret'] = 'secret'
+
+ @patch('forgeimporters.github.OAuth2Session')
+ @patch('forgeimporters.github.session')
+ def test_oauth_flow(self, session, oauth):
+ redirect = 'http://localhost/p/import_project/github/oauth_callback'
+ oauth_instance = Mock()
+ oauth_instance.authorization_url.return_value = (redirect, 'state')
+ oauth_instance.fetch_token.return_value = {'access_token': 'abc'}
+ oauth.return_value = oauth_instance
+
+ user = M.User.by_username('test-admin')
+ assert_equal(user.get_tool_data('GitHubProjectImport', 'token'), None)
+ r = self.app.get('/p/import_project/github/')
+ assert_equal(r.status_int, 302)
+ assert_equal(r.location, redirect)
+ session.__setitem__.assert_has_calls([
+ call('github.oauth.state', 'state'),
+ call('github.oauth.redirect', 'http://localhost/p/import_project/github/')
+ ])
+ session.save.assert_called_once()
+
+ r = self.app.get(redirect)
+ session.get.assert_has_calls([
+ call('github.oauth.state'),
+ call('github.oauth.redirect', '/')
+ ])
+ user = M.User.by_username('test-admin')
+ assert_equal(user.get_tool_data('GitHubProjectImport', 'token'), 'abc')
+
+ r = self.app.get('/p/import_project/github/')
+ assert_equal(r.status_int, 200) # token in user data, so oauth isn't triggered
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5f434893/ForgeImporters/forgeimporters/tests/github/test_tasks.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_tasks.py b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
index 86b321e..cf58644 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_tasks.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_tasks.py
@@ -25,8 +25,9 @@ from ...github import tasks
@mock.patch.object(tasks, 'c')
def test_import_project_info(c, session, ghpe):
c.project = mock.Mock(name='project')
+ c.user = mock.Mock(name='user')
tasks.import_project_info('my-project')
- ghpe.assert_called_once_with('my-project')
+ ghpe.assert_called_once_with('my-project', user=c.user)
ghpe.return_value.get_summary.assert_called_once_with()
ghpe.return_value.get_homepage.assert_called_once_with()
session.flush_all.assert_called_once_with()