You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/11/13 23:44:58 UTC
[2/8] git commit: [#6656] ticket:437 Check rate limit values and
sleep until reset
[#6656] ticket:437 Check rate limit values and sleep until reset
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/4d518a0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/4d518a0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/4d518a0f
Branch: refs/heads/master
Commit: 4d518a0f2c2e977d46f2cce6b7b81b82e6097516
Parents: d396c9e
Author: Igor Bondarenko <je...@gmail.com>
Authored: Fri Oct 25 12:53:52 2013 +0300
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Wed Nov 13 20:43:45 2013 +0000
----------------------------------------------------------------------
.../forgeimporters/github/__init__.py | 16 ++++++++--
.../tests/github/test_extractor.py | 31 ++++++++++++++++++++
2 files changed, 45 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4d518a0f/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 5726326..4932697 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -18,6 +18,8 @@
import re
import logging
import json
+import time
+from datetime import datetime
from forgeimporters import base
@@ -48,8 +50,18 @@ class GitHubProjectExtractor(base.ProjectExtractor):
return url
def urlopen(self, url, **kw):
- url = self.add_token(url)
- return super(GitHubProjectExtractor, self).urlopen(url, **kw)
+ resp = super(GitHubProjectExtractor, self).urlopen(self.add_token(url), **kw)
+ remain = resp.info().get('X-RateLimit-Remaining')
+ if remain and int(remain) == 0:
+ reset = resp.info().get('X-RateLimit-Reset')
+ limit = resp.info().get('X-RateLimit-Limit')
+ reset = datetime.utcfromtimestamp(int(reset))
+ now = datetime.utcnow()
+ log.warn('Rate limit exceeded (%s requests/hour). '
+ 'Sleeping until %s UTC' % (limit, reset))
+ time.sleep((reset - now).total_seconds())
+ self.urlopen(url, **kw)
+ return resp
def get_next_page_url(self, link):
if not link:
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/4d518a0f/ForgeImporters/forgeimporters/tests/github/test_extractor.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index a26438a..b3ad2f2 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -16,6 +16,7 @@
# under the License.
import json
+from datetime import datetime
from unittest import TestCase
from mock import patch, Mock
@@ -149,3 +150,33 @@ class TestGitHubProjectExtractor(TestCase):
e.urlopen(url)
request = urlopen.call_args[0][0]
self.assertEqual(request.get_full_url(), url + '&access_token=abc')
+
+ @patch('forgeimporters.base.h.urlopen')
+ @patch('forgeimporters.github.time.sleep')
+ @patch('forgeimporters.github.log')
+ def test_urlopen_rate_limit(self, log, sleep, urlopen):
+ limit_exceeded_headers = {
+ 'X-RateLimit-Limit': '10',
+ 'X-RateLimit-Remaining': '0',
+ 'X-RateLimit-Reset': '1382693522',
+ }
+ response_limit_exceeded = StringIO('{}')
+ response_limit_exceeded.info = lambda: limit_exceeded_headers
+ response_ok = StringIO('{}')
+ response_ok.info = lambda: {}
+ results = [response_limit_exceeded, response_ok]
+ urlopen.side_effect = lambda *a, **kw: results.pop(0)
+ e = github.GitHubProjectExtractor('test_project')
+ e.get_page('fake')
+ self.assertEqual(sleep.call_count, 1)
+ self.assertEqual(urlopen.call_count, 2)
+ log.warn.assert_called_once_with(
+ 'Rate limit exceeded (10 requests/hour). '
+ 'Sleeping until 2013-10-25 09:32:02 UTC'
+ )
+ sleep.reset_mock(); urlopen.reset_mock(); log.warn.reset_mock()
+ urlopen.side_effect = lambda *a, **kw: response_ok
+ e.get_page('fake 2')
+ self.assertEqual(sleep.call_count, 0)
+ self.assertEqual(urlopen.call_count, 1)
+ self.assertEqual(log.warn.call_count, 0)