You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2015/08/18 17:24:21 UTC
allura git commit: [#7970] expand HTTP codes for retry,
and retry all types of socket errors (they subclass IOError)
Repository: allura
Updated Branches:
refs/heads/db/7970 [created] 6aa2289f3
[#7970] expand HTTP codes for retry, and retry all types of socket errors (they subclass IOError)
Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/6aa2289f
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/6aa2289f
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/6aa2289f
Branch: refs/heads/db/7970
Commit: 6aa2289f36d9548b3f398c8d6d6e16f322a6da53
Parents: 7c5a5de
Author: Dave Brondsema <db...@slashdotmedia.com>
Authored: Tue Aug 18 15:23:25 2015 +0000
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Tue Aug 18 15:23:25 2015 +0000
----------------------------------------------------------------------
Allura/allura/lib/helpers.py | 10 +++++-----
Allura/allura/tests/test_helpers.py | 11 +++++++++++
ForgeImporters/forgeimporters/base.py | 2 +-
3 files changed, 17 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/allura/blob/6aa2289f/Allura/allura/lib/helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index 3a85c19..68330ba 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -1021,10 +1021,10 @@ class exceptionless(object):
return inner
-def urlopen(url, retries=3, codes=(408,), timeout=None):
+def urlopen(url, retries=3, codes=(408, 500, 502, 503, 504), timeout=None):
"""Open url, optionally retrying if an error is encountered.
- Socket timeouts will always be retried if retries > 0.
+ Socket and other IO errors will always be retried if retries > 0.
HTTP errors are retried if the error code is passed in ``codes``.
:param retries: Number of time to retry.
@@ -1035,9 +1035,9 @@ def urlopen(url, retries=3, codes=(408,), timeout=None):
while True:
try:
return urllib2.urlopen(url, timeout=timeout)
- except (urllib2.HTTPError, socket.timeout) as e:
- if attempts < retries and (isinstance(e, socket.timeout) or
- e.code in codes):
+ except IOError as e:
+ no_retry = isinstance(e, urllib2.HTTPError) and e.code not in codes
+ if attempts < retries and not no_retry:
attempts += 1
continue
else:
http://git-wip-us.apache.org/repos/asf/allura/blob/6aa2289f/Allura/allura/tests/test_helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 22d5b94..e9d13d2 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -449,6 +449,17 @@ class TestUrlOpen(TestCase):
self.assertEqual(urlopen.call_count, 4)
@patch('allura.lib.helpers.urllib2.urlopen')
+ def test_socket_reset(self, urlopen):
+ import socket
+ import errno
+
+ def side_effect(url, timeout=None):
+ raise socket.error(errno.ECONNRESET, 'Connection reset by peer')
+ urlopen.side_effect = side_effect
+ self.assertRaises(socket.error, h.urlopen, 'myurl')
+ self.assertEqual(urlopen.call_count, 4)
+
+ @patch('allura.lib.helpers.urllib2.urlopen')
def test_handled_http_error(self, urlopen):
from urllib2 import HTTPError
http://git-wip-us.apache.org/repos/asf/allura/blob/6aa2289f/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 8428142..8cc7e72 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -166,7 +166,7 @@ class ProjectExtractor(object):
self.get_page(page_name, **kw)
@staticmethod
- def urlopen(url, retries=3, codes=(408,), **kw):
+ def urlopen(url, retries=3, codes=(408, 500, 502, 503, 504), **kw):
req = urllib2.Request(url, **kw)
req.add_header(
'User-Agent', 'Allura Data Importer (https://allura.apache.org/)')