[kibble-scanners] branch master updated: Try our best to respect and compromise on rate limiting, quietly fail otherwise

[hu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

humbedooh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kibble-scanners.git


The following commit(s) were added to refs/heads/master by this push:
     new e6a4f86  Try our best to respect and compromise on rate limiting, quietly fail otherwise
e6a4f86 is described below

commit e6a4f86ee439a85f19c41d21576156f774983f71
Author: Daniel Gruno <hu...@apache.org>
AuthorDate: Thu Jan 24 22:17:10 2019 +0100

    Try our best to respect and compromise on rate limiting, quietly fail otherwise
---
 src/plugins/utils/jsonapi.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/plugins/utils/jsonapi.py b/src/plugins/utils/jsonapi.py
index 5687156..7449e0d 100644
--- a/src/plugins/utils/jsonapi.py
+++ b/src/plugins/utils/jsonapi.py
@@ -24,7 +24,7 @@ import time
 import re
 import base64
 
-def get(url, cookie = None, auth = None, token = None):
+def get(url, cookie = None, auth = None, token = None, retries = 5):
     headers = {
         "Content-type": "application/json",
         "Accept": "application/json"
@@ -39,6 +39,13 @@ def get(url, cookie = None, auth = None, token = None):
         headers["Cookie"] = cookie
     rv = requests.get(url, headers = headers)
     js = rv.json()
+    # Some services may be rate limited. We'll try sleeping it off in 60 second
+    # intervals for a max of five minutes, then give up.
+    if rv.status_code == 429:
+        if retries > 0:
+            time.sleep(60)
+            retries -= 1
+            return get(url, cookie = cookie, auth = auth, token = token, retries = retries)
     if rv.status_code != 404:
         return js
     return None