You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/11/14 16:37:23 UTC
[31/41] git commit: [#6656] ticket:437 More careful rate limit
checking
[#6656] ticket:437 More careful rate limit checking
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/3c07c836
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/3c07c836
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/3c07c836
Branch: refs/heads/cj/6845
Commit: 3c07c83643389b4402fb2c9b830ce7f835114517
Parents: 4d518a0
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 16:14:50 2013 +0300
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:43:45 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/github/__init__.py | 32 +++++++++++++-----
.../tests/github/test_extractor.py | 35 +++++++++++++++++---
2 files changed, 54 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c07c836/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 4932697..951d670 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -19,6 +19,7 @@ import re
import logging
import json
import time
+import urllib2
from datetime import datetime
from forgeimporters import base
@@ -49,18 +50,31 @@ class GitHubProjectExtractor(base.ProjectExtractor):
url += glue + 'access_token=' + self.token
return url
+ def wait_for_limit_reset(self, headers):
+ reset = headers.get('X-RateLimit-Reset')
+ limit = headers.get('X-RateLimit-Limit')
+ reset = datetime.utcfromtimestamp(int(reset))
+ now = datetime.utcnow()
+ log.warn('Rate limit exceeded (%s requests/hour). '
+ 'Sleeping until %s UTC' % (limit, reset))
+ time.sleep((reset - now).total_seconds())
+
def urlopen(self, url, **kw):
- resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+ try:
+ resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+ except urllib2.HTTPError as e:
+ # GitHub will return 403 if rate limit exceeded.
+ # We're checking for limit on every request below, but we still
+ # can get 403 if other import task exceeds the limit before.
+ if e.code == 403 and e.info().get('X-RateLimit-Remaining') == '0':
+ self.wait_for_limit_reset(e.info())
+ return self.urlopen(url, **kw)
+ else:
+ raise e
remain = resp.info().get('X-RateLimit-Remaining')
if remain and int(remain) == 0:
- reset = resp.info().get('X-RateLimit-Reset')
- limit = resp.info().get('X-RateLimit-Limit')
- reset = datetime.utcfromtimestamp(int(reset))
- now = datetime.utcnow()
- log.warn('Rate limit exceeded (%s requests/hour). '
- 'Sleeping until %s UTC' % (limit, reset))
- time.sleep((reset - now).total_seconds())
- self.urlopen(url, **kw)
+ self.wait_for_limit_reset(resp.info())
+ return self.urlopen(url, **kw)
return resp
def get_next_page_url(self, link):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/3c07c836/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index b3ad2f2..e6c96e3 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -16,8 +16,8 @@
# under the License.
import json
-from datetime import datetime
from unittest import TestCase
+import urllib2
from mock import patch, Mock
@@ -164,8 +164,7 @@ class TestGitHubProjectExtractor(TestCase):
response_limit_exceeded.info = lambda: limit_exceeded_headers
response_ok = StringIO('{}')
response_ok.info = lambda: {}
- results = [response_limit_exceeded, response_ok]
- urlopen.side_effect = lambda *a, **kw: results.pop(0)
+ urlopen.side_effect = [response_limit_exceeded, response_ok]
e = github.GitHubProjectExtractor('test_project')
e.get_page('fake')
self.assertEqual(sleep.call_count, 1)
@@ -175,8 +174,36 @@ class TestGitHubProjectExtractor(TestCase):
'Sleeping until 2013-10-25 09:32:02 UTC'
)
sleep.reset_mock(); urlopen.reset_mock(); log.warn.reset_mock()
- urlopen.side_effect = lambda *a, **kw: response_ok
+ response_ok = StringIO('{}')
+ response_ok.info = lambda: {}
+ urlopen.side_effect = [response_ok]
e.get_page('fake 2')
self.assertEqual(sleep.call_count, 0)
self.assertEqual(urlopen.call_count, 1)
self.assertEqual(log.warn.call_count, 0)
+
+ @patch('forgeimporters.base.h.urlopen')
+ @patch('forgeimporters.github.time.sleep')
+ @patch('forgeimporters.github.log')
+ def test_urlopen_rate_limit_403(self, log, sleep, urlopen):
+ '''Test that urlopen catches 403 which may happen if limit exceeded by another task'''
+ limit_exceeded_headers = {
+ 'X-RateLimit-Limit': '10',
+ 'X-RateLimit-Remaining': '0',
+ 'X-RateLimit-Reset': '1382693522',
+ }
+ def urlopen_side_effect(*a, **kw):
+ mock_resp = StringIO('{}')
+ mock_resp.info = lambda: {}
+ urlopen.side_effect = [mock_resp]
+ raise urllib2.HTTPError(
+ 'url', 403, 'msg', limit_exceeded_headers, StringIO('{}'))
+ urlopen.side_effect = urlopen_side_effect
+ e = github.GitHubProjectExtractor('test_project')
+ e.get_page('fake')
+ self.assertEqual(sleep.call_count, 1)
+ self.assertEqual(urlopen.call_count, 2)
+ log.warn.assert_called_once_with(
+ 'Rate limit exceeded (10 requests/hour). '
+ 'Sleeping until 2013-10-25 09:32:02 UTC'
+ )