You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by gc...@apache.org on 2023/10/23 16:11:06 UTC
[allura] branch master updated (e97f3c389 -> 2a8bc148f)
This is an automated email from the ASF dual-hosted git repository.
gcruz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
from e97f3c389 [#8522] bumped urlib3 2.0.6 -> 2.0.7
new 910da1620 [#8523] use Bearer instead of token in auth header
new 3624d8c6a [#8523] let caller control use_auth_headers_on_redirects
new 5935ae829 [#8523] never proactively sleep after a request (and then request again); only sleep if this request got limited
new 2a8bc148f [#8523] random type hint
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Allura/allura/model/project.py | 3 +-
ForgeImporters/forgeimporters/github/__init__.py | 36 +++++++++++----------
.../forgeimporters/tests/github/test_extractor.py | 37 ++--------------------
3 files changed, 24 insertions(+), 52 deletions(-)
[allura] 01/04: [#8523] use Bearer instead of token in auth header
Posted by gc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gcruz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 910da16200248a7f94420f55cacf6d7d6b1ecc04
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Oct 18 14:44:31 2023 -0400
[#8523] use Bearer instead of token in auth header
---
ForgeImporters/forgeimporters/github/__init__.py | 2 +-
ForgeImporters/forgeimporters/tests/github/test_extractor.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 3b01a6224..9b82a5998 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -89,7 +89,7 @@ class GitHubProjectExtractor(base.ProjectExtractor):
def add_token(self, url):
headers = {}
if self.token:
- headers['Authorization'] = f'token {self.token}'
+ headers['Authorization'] = f'Bearer {self.token}'
return url, headers
def wait_for_limit_reset(self, headers):
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index 4d3a022cf..6b5541574 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -149,7 +149,7 @@ class TestGitHubProjectExtractor(TestCase):
request = urlopen.call_args[0][0]
self.assertEqual(request.get_full_url(), url)
assert request.headers['User-agent']
- self.assertEqual(request.unredirected_hdrs['Authorization'], 'token abc')
+ self.assertEqual(request.unredirected_hdrs['Authorization'], 'Bearer abc')
@patch('forgeimporters.base.h.urlopen')
@patch('forgeimporters.github.time.sleep')
[allura] 04/04: [#8523] random type hint
Posted by gc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gcruz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 2a8bc148f3c92067b884d8f13539443fff1bdcdb
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Oct 18 18:25:06 2023 -0400
[#8523] random type hint
---
Allura/allura/model/project.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Allura/allura/model/project.py b/Allura/allura/model/project.py
index 63b7e413a..33a675e8a 100644
--- a/Allura/allura/model/project.py
+++ b/Allura/allura/model/project.py
@@ -71,6 +71,7 @@ import six
if typing.TYPE_CHECKING:
from ming.odm.mapper import Query
from allura.model import AppConfig
+ from allura.app import Application
log = logging.getLogger(__name__)
@@ -1440,7 +1441,7 @@ class AppConfig(MappedClass, ActivityObject):
'''ACL processing should terminate at the AppConfig'''
return None
- def load(self):
+ def load(self) -> typing.Type[Application]:
"""
:returns: the related :class:`Application <allura.app.Application>` class
"""
[allura] 02/04: [#8523] let caller control use_auth_headers_on_redirects
Posted by gc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gcruz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 3624d8c6a9345bfe3404b4f449247a8ad196a42b
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Oct 18 18:23:13 2023 -0400
[#8523] let caller control use_auth_headers_on_redirects
---
ForgeImporters/forgeimporters/github/__init__.py | 24 +++++++++++++++-------
.../forgeimporters/tests/github/test_extractor.py | 2 +-
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 9b82a5998..b7d08b0d8 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -101,22 +101,32 @@ class GitHubProjectExtractor(base.ProjectExtractor):
'Sleeping until %s UTC' % (limit, reset))
time.sleep((reset - now).total_seconds() + 2)
- def urlopen(self, url, headers=None, **kw):
+ def urlopen(self, url, headers=None, use_auth_headers_on_redirects=True, **kw):
+ """
+ :param url: the URL
+ :param headers: dict of headers
+ :param use_auth_headers_on_redirects: in some cases (assets on AWS) you need to set this to False
+ :param kw: extra args to urlopen
+ """
if headers is None:
headers = {}
+ url, auth_headers = self.add_token(url)
+ if use_auth_headers_on_redirects:
+ headers.update(auth_headers)
+ unredirected_hdrs = {}
+ else:
+ unredirected_hdrs = auth_headers
try:
- url, auth_headers = self.add_token(url)
- # need to use unredirected_hdrs for Authorization for APIs that redirect to an AWS file asset which has
- # separate authentication added automatically
- resp = super().urlopen(url,
- headers=headers, unredirected_hdrs=auth_headers, **kw)
+ resp = super().urlopen(url, headers=headers, unredirected_hdrs=unredirected_hdrs, **kw)
except six.moves.urllib.error.HTTPError as e:
# GitHub will return 403 if rate limit exceeded.
# We're checking for limit on every request below, but we still
# can get 403 if other import task exceeds the limit before.
if e.code == 403 and e.info().get('X-RateLimit-Remaining') == '0':
self.wait_for_limit_reset(e.info())
- return self.urlopen(url, **kw)
+ # call ourselves to try again:
+ return self.urlopen(url, headers=headers, use_auth_headers_on_redirects=use_auth_headers_on_redirects,
+ **kw)
else:
raise e
remain = resp.info().get('X-RateLimit-Remaining')
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index 6b5541574..3fd6b2f03 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -149,7 +149,7 @@ class TestGitHubProjectExtractor(TestCase):
request = urlopen.call_args[0][0]
self.assertEqual(request.get_full_url(), url)
assert request.headers['User-agent']
- self.assertEqual(request.unredirected_hdrs['Authorization'], 'Bearer abc')
+ self.assertEqual(request.headers['Authorization'], 'Bearer abc')
@patch('forgeimporters.base.h.urlopen')
@patch('forgeimporters.github.time.sleep')
[allura] 03/04: [#8523] never proactively sleep after a request (and then request again); only sleep if this request got limited
Posted by gc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gcruz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 5935ae82998e227a643f979cd3bc016e0fce7ea9
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Oct 18 18:24:34 2023 -0400
[#8523] never proactively sleep after a request (and then request again); only sleep if this request got limited
---
ForgeImporters/forgeimporters/github/__init__.py | 12 ++------
.../forgeimporters/tests/github/test_extractor.py | 35 +---------------------
2 files changed, 4 insertions(+), 43 deletions(-)
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index b7d08b0d8..f14d215f7 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -97,6 +97,7 @@ class GitHubProjectExtractor(base.ProjectExtractor):
limit = headers.get('X-RateLimit-Limit')
reset = datetime.utcfromtimestamp(int(reset))
now = datetime.utcnow()
+ # 60/hour is for GitHub unauthenticated users. If you get that, check your auth tokens
log.warning('Rate limit exceeded (%s requests/hour). '
'Sleeping until %s UTC' % (limit, reset))
time.sleep((reset - now).total_seconds() + 2)
@@ -117,23 +118,16 @@ class GitHubProjectExtractor(base.ProjectExtractor):
else:
unredirected_hdrs = auth_headers
try:
- resp = super().urlopen(url, headers=headers, unredirected_hdrs=unredirected_hdrs, **kw)
+ return super().urlopen(url, headers=headers, unredirected_hdrs=unredirected_hdrs, **kw)
except six.moves.urllib.error.HTTPError as e:
# GitHub will return 403 if rate limit exceeded.
- # We're checking for limit on every request below, but we still
- # can get 403 if other import task exceeds the limit before.
if e.code == 403 and e.info().get('X-RateLimit-Remaining') == '0':
self.wait_for_limit_reset(e.info())
# call ourselves to try again:
return self.urlopen(url, headers=headers, use_auth_headers_on_redirects=use_auth_headers_on_redirects,
**kw)
else:
- raise e
- remain = resp.info().get('X-RateLimit-Remaining')
- if remain and int(remain) == 0:
- self.wait_for_limit_reset(resp.info())
- return self.urlopen(url, **kw)
- return resp
+ raise
def check_readable(self):
url, headers = self.add_token(self.get_page_url('project_info'))
diff --git a/ForgeImporters/forgeimporters/tests/github/test_extractor.py b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
index 3fd6b2f03..e33faf357 100644
--- a/ForgeImporters/forgeimporters/tests/github/test_extractor.py
+++ b/ForgeImporters/forgeimporters/tests/github/test_extractor.py
@@ -151,44 +151,11 @@ class TestGitHubProjectExtractor(TestCase):
assert request.headers['User-agent']
self.assertEqual(request.headers['Authorization'], 'Bearer abc')
- @patch('forgeimporters.base.h.urlopen')
- @patch('forgeimporters.github.time.sleep')
- @patch('forgeimporters.github.log')
- def test_urlopen_rate_limit(self, log, sleep, urlopen):
- limit_exceeded_headers = {
- 'X-RateLimit-Limit': '10',
- 'X-RateLimit-Remaining': '0',
- 'X-RateLimit-Reset': '1382693522',
- }
- response_limit_exceeded = BytesIO(b'{}')
- response_limit_exceeded.info = lambda: limit_exceeded_headers
- response_ok = BytesIO(b'{}')
- response_ok.info = lambda: {}
- urlopen.side_effect = [response_limit_exceeded, response_ok]
- e = github.GitHubProjectExtractor('test_project')
- e.get_page('http://example.com/')
- self.assertEqual(sleep.call_count, 1)
- self.assertEqual(urlopen.call_count, 2)
- log.warning.assert_called_once_with(
- 'Rate limit exceeded (10 requests/hour). '
- 'Sleeping until 2013-10-25 09:32:02 UTC'
- )
- sleep.reset_mock()
- urlopen.reset_mock()
- log.warning.reset_mock()
- response_ok = BytesIO(b'{}')
- response_ok.info = lambda: {}
- urlopen.side_effect = [response_ok]
- e.get_page('http://example.com/2')
- self.assertEqual(sleep.call_count, 0)
- self.assertEqual(urlopen.call_count, 1)
- self.assertEqual(log.warning.call_count, 0)
-
@patch('forgeimporters.base.h.urlopen')
@patch('forgeimporters.github.time.sleep')
@patch('forgeimporters.github.log')
def test_urlopen_rate_limit_403(self, log, sleep, urlopen):
- '''Test that urlopen catches 403 which may happen if limit exceeded by another task'''
+ '''Test that urlopen catches 403 which may happen if limit exceeded by previous fetches'''
limit_exceeded_headers = {
'X-RateLimit-Limit': '10',
'X-RateLimit-Remaining': '0',