[kibble-scanners] 01/03: tweak GitHub to work around abuse limitations

[hu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble-scanners.git

commit ba6a37aa06d79b88636b72ba90e380cacc80585f
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Wed Jul 17 05:50:30 2019 +0200

    tweak GitHub to work around abuse limitations
    
    If we hit the abuse detection system, try to sleep it off a few times
    before finally bailing.
---
 src/plugins/utils/github.py | 52 +++++++++++++++++++++++++++------------------
 1 file changed, 31 insertions(+), 21 deletions(-)

diff --git a/src/plugins/utils/github.py b/src/plugins/utils/github.py
index eb0389b..6b979a9 100644
--- a/src/plugins/utils/github.py
+++ b/src/plugins/utils/github.py
@@ -25,50 +25,60 @@ repo_pattern = re.compile('.*[:/]([^/]+)/([^/]+).git')
 issues_api = "https://api.github.com/repos/%s/%s/issues"
 traffic_api = "https://api.github.com/repos/%s/%s/traffic"
 popular_api = "https://api.github.com/repos/%s/%s/popular"
+rate_limit_api = "https://api.github.com/rate_limit"
+
+def get_limited(url, params = None, auth = None):
+    """ Get a GitHub API response, keeping in mind that we may
+        be rate-limited by the abuse system """
+    number_of_retries = 0
+    resp = requests.get(url, auth=auth)
+    while resp.status_code == 403 and number_of_retries < 20:
+        js = resp.json()
+        # If abuse-detection kicks in, sleep it off
+        if 'You have triggered an abuse' in js['message']:
+            time.sleep(5)
+            number_of_retries += 1
+            resp = requests.get(url, auth=auth)
+        else:
+            break
+    resp.raise_for_status()
+    return resp.json()
+
+def get_tokens_left(auth = None):
+    """ Gets number of GitHub tokens left this hour... """
+    js = get_limited(rate_limit_api, auth=auth)
+    tokens_left = js['rate']['remaining']
+    return tokens_left
+
 
 def issues(source, params={}, auth=None):
     local_params = {'per_page': 100, 'page': 1}
     local_params.update(params)
 
     repo_user = repo_pattern.findall(source['sourceURL'])[0]
-    resp = requests.get(issues_api % repo_user, params=local_params, auth=auth)
-    resp.raise_for_status()
-
-    return resp.json()
+    return get_limited(issues_api % repo_user, params=local_params, auth=auth)
 
 def views(source, auth=None):
     repo_user = repo_pattern.findall(source['sourceURL'])[0]
-    resp = requests.get("%s/views" % (traffic_api % repo_user), auth=auth)
-    resp.raise_for_status()
-    
-    return resp.json()
+    return get_limited("%s/views" % (traffic_api % repo_user), auth=auth)
 
 def clones(source, auth=None):
     repo_user = repo_pattern.findall(source['sourceURL'])[0]
-    resp = requests.get("%s/clones" % (traffic_api % repo_user), auth=auth)
-    resp.raise_for_status()
-    
-    return resp.json()
+    return get_limited("%s/clones" % (traffic_api % repo_user), auth=auth)
 
 def referrers(source, auth=None):
     repo_user = repo_pattern.findall(source['sourceURL'])[0]
-    resp = requests.get("%s/referrers" % (popular_api % repo_user), auth=auth)
-    resp.raise_for_status()
+    return get_limited("%s/referrers" % (popular_api % repo_user), auth=auth)
     
-    return resp.json()
-
 def user(user_url, auth=None):
-    resp = requests.get(user_url, auth=auth)
-    resp.raise_for_status()
-
-    return resp.json()
+    return get_limited(user_url, auth=auth)
 
 def get_all(source, f, params={}, auth=None):
     acc = []
     page = params.get('page', 1)
 
     while True:
-        time.sleep(1.25)
+        time.sleep(1.5)
         items = f(source, params=params, auth=auth)
         if not items:
             break